LCOV - code coverage report
Current view: top level - drivers/usb/host - xhci.c (source / functions) Hit Total Coverage
Test: combined.info Lines: 22 2130 1.0 %
Date: 2022-03-28 16:04:14 Functions: 2 80 2.5 %
Branches: 5 1088 0.5 %

           Branch data     Line data    Source code
       1                 :            : // SPDX-License-Identifier: GPL-2.0
       2                 :            : /*
       3                 :            :  * xHCI host controller driver
       4                 :            :  *
       5                 :            :  * Copyright (C) 2008 Intel Corp.
       6                 :            :  *
       7                 :            :  * Author: Sarah Sharp
       8                 :            :  * Some code borrowed from the Linux EHCI driver.
       9                 :            :  */
      10                 :            : 
      11                 :            : #include <linux/pci.h>
      12                 :            : #include <linux/iopoll.h>
      13                 :            : #include <linux/irq.h>
      14                 :            : #include <linux/log2.h>
      15                 :            : #include <linux/module.h>
      16                 :            : #include <linux/moduleparam.h>
      17                 :            : #include <linux/slab.h>
      18                 :            : #include <linux/dmi.h>
      19                 :            : #include <linux/dma-mapping.h>
      20                 :            : 
      21                 :            : #include "xhci.h"
      22                 :            : #include "xhci-trace.h"
      23                 :            : #include "xhci-mtk.h"
      24                 :            : #include "xhci-debugfs.h"
      25                 :            : #include "xhci-dbgcap.h"
      26                 :            : 
      27                 :            : #define DRIVER_AUTHOR "Sarah Sharp"
      28                 :            : #define DRIVER_DESC "'eXtensible' Host Controller (xHC) Driver"
      29                 :            : 
      30                 :            : #define PORT_WAKE_BITS  (PORT_WKOC_E | PORT_WKDISC_E | PORT_WKCONN_E)
      31                 :            : 
      32                 :            : /* Some 0.95 hardware can't handle the chain bit on a Link TRB being cleared */
      33                 :            : static int link_quirk;
      34                 :            : module_param(link_quirk, int, S_IRUGO | S_IWUSR);
      35                 :            : MODULE_PARM_DESC(link_quirk, "Don't clear the chain bit on a link TRB");
      36                 :            : 
      37                 :            : static unsigned long long quirks;
      38                 :            : module_param(quirks, ullong, S_IRUGO);
      39                 :            : MODULE_PARM_DESC(quirks, "Bit flags for quirks to be enabled as default");
      40                 :            : 
      41                 :          0 : static bool td_on_ring(struct xhci_td *td, struct xhci_ring *ring)
      42                 :            : {
      43                 :          0 :         struct xhci_segment *seg = ring->first_seg;
      44                 :            : 
      45         [ #  # ]:          0 :         if (!td || !td->start_seg)
      46                 :            :                 return false;
      47                 :          0 :         do {
      48         [ #  # ]:          0 :                 if (seg == td->start_seg)
      49                 :            :                         return true;
      50                 :          0 :                 seg = seg->next;
      51   [ #  #  #  # ]:          0 :         } while (seg && seg != ring->first_seg);
      52                 :            : 
      53                 :            :         return false;
      54                 :            : }
      55                 :            : 
      56                 :            : /*
      57                 :            :  * xhci_handshake - spin reading hc until handshake completes or fails
      58                 :            :  * @ptr: address of hc register to be read
      59                 :            :  * @mask: bits to look at in result of read
      60                 :            :  * @done: value of those bits when handshake succeeds
      61                 :            :  * @usec: timeout in microseconds
      62                 :            :  *
      63                 :            :  * Returns negative errno, or zero on success
      64                 :            :  *
      65                 :            :  * Success happens when the "mask" bits have the specified value (hardware
      66                 :            :  * handshake done).  There are two failure modes:  "usec" have passed (major
      67                 :            :  * hardware flakeout), or the register reads as all-ones (hardware removed).
      68                 :            :  */
      69                 :          0 : int xhci_handshake(void __iomem *ptr, u32 mask, u32 done, int usec)
      70                 :            : {
      71                 :          0 :         u32     result;
      72                 :          0 :         int     ret;
      73                 :            : 
      74   [ #  #  #  #  :          0 :         ret = readl_poll_timeout_atomic(ptr, result,
          #  #  #  #  #  
                      # ]
      75                 :            :                                         (result & mask) == done ||
      76                 :            :                                         result == U32_MAX,
      77                 :            :                                         1, usec);
      78         [ #  # ]:          0 :         if (result == U32_MAX)          /* card removed */
      79                 :          0 :                 return -ENODEV;
      80                 :            : 
      81                 :            :         return ret;
      82                 :            : }
      83                 :            : 
      84                 :            : /*
      85                 :            :  * Disable interrupts and begin the xHCI halting process.
      86                 :            :  */
      87                 :          0 : void xhci_quiesce(struct xhci_hcd *xhci)
      88                 :            : {
      89                 :          0 :         u32 halted;
      90                 :          0 :         u32 cmd;
      91                 :          0 :         u32 mask;
      92                 :            : 
      93                 :          0 :         mask = ~(XHCI_IRQS);
      94                 :          0 :         halted = readl(&xhci->op_regs->status) & STS_HALT;
      95                 :          0 :         if (!halted)
      96                 :            :                 mask &= ~CMD_RUN;
      97                 :            : 
      98                 :          0 :         cmd = readl(&xhci->op_regs->command);
      99                 :          0 :         cmd &= mask;
     100                 :          0 :         writel(cmd, &xhci->op_regs->command);
     101                 :          0 : }
     102                 :            : 
     103                 :            : /*
     104                 :            :  * Force HC into halt state.
     105                 :            :  *
     106                 :            :  * Disable any IRQs and clear the run/stop bit.
     107                 :            :  * HC will complete any current and actively pipelined transactions, and
     108                 :            :  * should halt within 16 ms of the run/stop bit being cleared.
     109                 :            :  * Read HC Halted bit in the status register to see when the HC is finished.
     110                 :            :  */
     111                 :          0 : int xhci_halt(struct xhci_hcd *xhci)
     112                 :            : {
     113                 :          0 :         int ret;
     114                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Halt the HC");
     115                 :          0 :         xhci_quiesce(xhci);
     116                 :            : 
     117                 :          0 :         ret = xhci_handshake(&xhci->op_regs->status,
     118                 :            :                         STS_HALT, STS_HALT, XHCI_MAX_HALT_USEC);
     119         [ #  # ]:          0 :         if (ret) {
     120                 :          0 :                 xhci_warn(xhci, "Host halt failed, %d\n", ret);
     121                 :          0 :                 return ret;
     122                 :            :         }
     123                 :          0 :         xhci->xhc_state |= XHCI_STATE_HALTED;
     124                 :          0 :         xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
     125                 :          0 :         return ret;
     126                 :            : }
     127                 :            : 
     128                 :            : /*
     129                 :            :  * Set the run bit and wait for the host to be running.
     130                 :            :  */
     131                 :          0 : int xhci_start(struct xhci_hcd *xhci)
     132                 :            : {
     133                 :          0 :         u32 temp;
     134                 :          0 :         int ret;
     135                 :            : 
     136                 :          0 :         temp = readl(&xhci->op_regs->command);
     137                 :          0 :         temp |= (CMD_RUN);
     138                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Turn on HC, cmd = 0x%x.",
     139                 :            :                         temp);
     140                 :          0 :         writel(temp, &xhci->op_regs->command);
     141                 :            : 
     142                 :            :         /*
     143                 :            :          * Wait for the HCHalted Status bit to be 0 to indicate the host is
     144                 :            :          * running.
     145                 :            :          */
     146                 :          0 :         ret = xhci_handshake(&xhci->op_regs->status,
     147                 :            :                         STS_HALT, 0, XHCI_MAX_HALT_USEC);
     148         [ #  # ]:          0 :         if (ret == -ETIMEDOUT)
     149                 :          0 :                 xhci_err(xhci, "Host took too long to start, "
     150                 :            :                                 "waited %u microseconds.\n",
     151                 :            :                                 XHCI_MAX_HALT_USEC);
     152         [ #  # ]:          0 :         if (!ret)
     153                 :            :                 /* clear state flags. Including dying, halted or removing */
     154                 :          0 :                 xhci->xhc_state = 0;
     155                 :            : 
     156                 :          0 :         return ret;
     157                 :            : }
     158                 :            : 
     159                 :            : /*
     160                 :            :  * Reset a halted HC.
     161                 :            :  *
     162                 :            :  * This resets pipelines, timers, counters, state machines, etc.
     163                 :            :  * Transactions will be terminated immediately, and operational registers
     164                 :            :  * will be set to their defaults.
     165                 :            :  */
     166                 :          0 : int xhci_reset(struct xhci_hcd *xhci)
     167                 :            : {
     168                 :          0 :         u32 command;
     169                 :          0 :         u32 state;
     170                 :          0 :         int ret;
     171                 :            : 
     172                 :          0 :         state = readl(&xhci->op_regs->status);
     173                 :            : 
     174         [ #  # ]:          0 :         if (state == ~(u32)0) {
     175                 :          0 :                 xhci_warn(xhci, "Host not accessible, reset failed.\n");
     176                 :          0 :                 return -ENODEV;
     177                 :            :         }
     178                 :            : 
     179         [ #  # ]:          0 :         if ((state & STS_HALT) == 0) {
     180                 :          0 :                 xhci_warn(xhci, "Host controller not halted, aborting reset.\n");
     181                 :          0 :                 return 0;
     182                 :            :         }
     183                 :            : 
     184                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "// Reset the HC");
     185                 :          0 :         command = readl(&xhci->op_regs->command);
     186                 :          0 :         command |= CMD_RESET;
     187                 :          0 :         writel(command, &xhci->op_regs->command);
     188                 :            : 
     189                 :            :         /* Existing Intel xHCI controllers require a delay of 1 mS,
     190                 :            :          * after setting the CMD_RESET bit, and before accessing any
     191                 :            :          * HC registers. This allows the HC to complete the
     192                 :            :          * reset operation and be ready for HC register access.
     193                 :            :          * Without this delay, the subsequent HC register access,
     194                 :            :          * may result in a system hang very rarely.
     195                 :            :          */
     196         [ #  # ]:          0 :         if (xhci->quirks & XHCI_INTEL_HOST)
     197                 :          0 :                 udelay(1000);
     198                 :            : 
     199                 :          0 :         ret = xhci_handshake(&xhci->op_regs->command,
     200                 :            :                         CMD_RESET, 0, 10 * 1000 * 1000);
     201         [ #  # ]:          0 :         if (ret)
     202                 :            :                 return ret;
     203                 :            : 
     204         [ #  # ]:          0 :         if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
     205                 :          0 :                 usb_asmedia_modifyflowcontrol(to_pci_dev(xhci_to_hcd(xhci)->self.controller));
     206                 :            : 
     207                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     208                 :            :                          "Wait for controller to be ready for doorbell rings");
     209                 :            :         /*
     210                 :            :          * xHCI cannot write to any doorbells or operational registers other
     211                 :            :          * than status until the "Controller Not Ready" flag is cleared.
     212                 :            :          */
     213                 :          0 :         ret = xhci_handshake(&xhci->op_regs->status,
     214                 :            :                         STS_CNR, 0, 10 * 1000 * 1000);
     215                 :            : 
     216                 :          0 :         xhci->usb2_rhub.bus_state.port_c_suspend = 0;
     217                 :          0 :         xhci->usb2_rhub.bus_state.suspended_ports = 0;
     218                 :          0 :         xhci->usb2_rhub.bus_state.resuming_ports = 0;
     219                 :          0 :         xhci->usb3_rhub.bus_state.port_c_suspend = 0;
     220                 :          0 :         xhci->usb3_rhub.bus_state.suspended_ports = 0;
     221                 :          0 :         xhci->usb3_rhub.bus_state.resuming_ports = 0;
     222                 :            : 
     223                 :          0 :         return ret;
     224                 :            : }
     225                 :            : 
     226                 :          0 : static void xhci_zero_64b_regs(struct xhci_hcd *xhci)
     227                 :            : {
     228         [ #  # ]:          0 :         struct device *dev = xhci_to_hcd(xhci)->self.sysdev;
     229                 :          0 :         int err, i;
     230                 :          0 :         u64 val;
     231                 :            : 
     232                 :            :         /*
     233                 :            :          * Some Renesas controllers get into a weird state if they are
     234                 :            :          * reset while programmed with 64bit addresses (they will preserve
     235                 :            :          * the top half of the address in internal, non visible
     236                 :            :          * registers). You end up with half the address coming from the
     237                 :            :          * kernel, and the other half coming from the firmware. Also,
     238                 :            :          * changing the programming leads to extra accesses even if the
     239                 :            :          * controller is supposed to be halted. The controller ends up with
     240                 :            :          * a fatal fault, and is then ripe for being properly reset.
     241                 :            :          *
     242                 :            :          * Special care is taken to only apply this if the device is behind
     243                 :            :          * an iommu. Doing anything when there is no iommu is definitely
     244                 :            :          * unsafe...
     245                 :            :          */
     246   [ #  #  #  # ]:          0 :         if (!(xhci->quirks & XHCI_ZERO_64B_REGS) || !device_iommu_mapped(dev))
     247                 :            :                 return;
     248                 :            : 
     249                 :          0 :         xhci_info(xhci, "Zeroing 64bit base registers, expecting fault\n");
     250                 :            : 
     251                 :            :         /* Clear HSEIE so that faults do not get signaled */
     252                 :          0 :         val = readl(&xhci->op_regs->command);
     253                 :          0 :         val &= ~CMD_HSEIE;
     254                 :          0 :         writel(val, &xhci->op_regs->command);
     255                 :            : 
     256                 :            :         /* Clear HSE (aka FATAL) */
     257                 :          0 :         val = readl(&xhci->op_regs->status);
     258                 :          0 :         val |= STS_FATAL;
     259                 :          0 :         writel(val, &xhci->op_regs->status);
     260                 :            : 
     261                 :            :         /* Now zero the registers, and brace for impact */
     262                 :          0 :         val = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
     263         [ #  # ]:          0 :         if (upper_32_bits(val))
     264                 :          0 :                 xhci_write_64(xhci, 0, &xhci->op_regs->dcbaa_ptr);
     265                 :          0 :         val = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
     266         [ #  # ]:          0 :         if (upper_32_bits(val))
     267                 :          0 :                 xhci_write_64(xhci, 0, &xhci->op_regs->cmd_ring);
     268                 :            : 
     269         [ #  # ]:          0 :         for (i = 0; i < HCS_MAX_INTRS(xhci->hcs_params1); i++) {
     270                 :          0 :                 struct xhci_intr_reg __iomem *ir;
     271                 :            : 
     272                 :          0 :                 ir = &xhci->run_regs->ir_set[i];
     273                 :          0 :                 val = xhci_read_64(xhci, &ir->erst_base);
     274         [ #  # ]:          0 :                 if (upper_32_bits(val))
     275                 :          0 :                         xhci_write_64(xhci, 0, &ir->erst_base);
     276                 :          0 :                 val= xhci_read_64(xhci, &ir->erst_dequeue);
     277         [ #  # ]:          0 :                 if (upper_32_bits(val))
     278                 :          0 :                         xhci_write_64(xhci, 0, &ir->erst_dequeue);
     279                 :            :         }
     280                 :            : 
     281                 :            :         /* Wait for the fault to appear. It will be cleared on reset */
     282                 :          0 :         err = xhci_handshake(&xhci->op_regs->status,
     283                 :            :                              STS_FATAL, STS_FATAL,
     284                 :            :                              XHCI_MAX_HALT_USEC);
     285         [ #  # ]:          0 :         if (!err)
     286                 :          0 :                 xhci_info(xhci, "Fault detected\n");
     287                 :            : }
     288                 :            : 
     289                 :            : #ifdef CONFIG_USB_PCI
     290                 :            : /*
     291                 :            :  * Set up MSI
     292                 :            :  */
     293                 :          0 : static int xhci_setup_msi(struct xhci_hcd *xhci)
     294                 :            : {
     295                 :          0 :         int ret;
     296                 :            :         /*
     297                 :            :          * TODO:Check with MSI Soc for sysdev
     298                 :            :          */
     299                 :          0 :         struct pci_dev  *pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
     300                 :            : 
     301                 :          0 :         ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI);
     302         [ #  # ]:          0 :         if (ret < 0) {
     303                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     304                 :            :                                 "failed to allocate MSI entry");
     305                 :          0 :                 return ret;
     306                 :            :         }
     307                 :            : 
     308                 :          0 :         ret = request_irq(pdev->irq, xhci_msi_irq,
     309                 :          0 :                                 0, "xhci_hcd", xhci_to_hcd(xhci));
     310         [ #  # ]:          0 :         if (ret) {
     311                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     312                 :            :                                 "disable MSI interrupt");
     313                 :          0 :                 pci_free_irq_vectors(pdev);
     314                 :            :         }
     315                 :            : 
     316                 :            :         return ret;
     317                 :            : }
     318                 :            : 
     319                 :            : /*
     320                 :            :  * Set up MSI-X
     321                 :            :  */
     322                 :          0 : static int xhci_setup_msix(struct xhci_hcd *xhci)
     323                 :            : {
     324                 :          0 :         int i, ret = 0;
     325                 :          0 :         struct usb_hcd *hcd = xhci_to_hcd(xhci);
     326                 :          0 :         struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
     327                 :            : 
     328                 :            :         /*
     329                 :            :          * calculate number of msi-x vectors supported.
     330                 :            :          * - HCS_MAX_INTRS: the max number of interrupts the host can handle,
     331                 :            :          *   with max number of interrupters based on the xhci HCSPARAMS1.
     332                 :            :          * - num_online_cpus: maximum msi-x vectors per CPUs core.
     333                 :            :          *   Add additional 1 vector to ensure always available interrupt.
     334                 :            :          */
     335                 :          0 :         xhci->msix_count = min(num_online_cpus() + 1,
     336                 :            :                                 HCS_MAX_INTRS(xhci->hcs_params1));
     337                 :            : 
     338                 :          0 :         ret = pci_alloc_irq_vectors(pdev, xhci->msix_count, xhci->msix_count,
     339                 :            :                         PCI_IRQ_MSIX);
     340         [ #  # ]:          0 :         if (ret < 0) {
     341                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     342                 :            :                                 "Failed to enable MSI-X");
     343                 :          0 :                 return ret;
     344                 :            :         }
     345                 :            : 
     346         [ #  # ]:          0 :         for (i = 0; i < xhci->msix_count; i++) {
     347                 :          0 :                 ret = request_irq(pci_irq_vector(pdev, i), xhci_msi_irq, 0,
     348                 :          0 :                                 "xhci_hcd", xhci_to_hcd(xhci));
     349         [ #  # ]:          0 :                 if (ret)
     350                 :          0 :                         goto disable_msix;
     351                 :            :         }
     352                 :            : 
     353                 :          0 :         hcd->msix_enabled = 1;
     354                 :          0 :         return ret;
     355                 :            : 
     356                 :            : disable_msix:
     357                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "disable MSI-X interrupt");
     358         [ #  # ]:          0 :         while (--i >= 0)
     359                 :          0 :                 free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
     360                 :          0 :         pci_free_irq_vectors(pdev);
     361                 :          0 :         return ret;
     362                 :            : }
     363                 :            : 
     364                 :            : /* Free any IRQs and disable MSI-X */
     365                 :          0 : static void xhci_cleanup_msix(struct xhci_hcd *xhci)
     366                 :            : {
     367         [ #  # ]:          0 :         struct usb_hcd *hcd = xhci_to_hcd(xhci);
     368                 :          0 :         struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
     369                 :            : 
     370         [ #  # ]:          0 :         if (xhci->quirks & XHCI_PLAT)
     371                 :            :                 return;
     372                 :            : 
     373                 :            :         /* return if using legacy interrupt */
     374         [ #  # ]:          0 :         if (hcd->irq > 0)
     375                 :            :                 return;
     376                 :            : 
     377         [ #  # ]:          0 :         if (hcd->msix_enabled) {
     378                 :            :                 int i;
     379                 :            : 
     380         [ #  # ]:          0 :                 for (i = 0; i < xhci->msix_count; i++)
     381                 :          0 :                         free_irq(pci_irq_vector(pdev, i), xhci_to_hcd(xhci));
     382                 :            :         } else {
     383                 :          0 :                 free_irq(pci_irq_vector(pdev, 0), xhci_to_hcd(xhci));
     384                 :            :         }
     385                 :            : 
     386                 :          0 :         pci_free_irq_vectors(pdev);
     387                 :          0 :         hcd->msix_enabled = 0;
     388                 :            : }
     389                 :            : 
     390                 :          0 : static void __maybe_unused xhci_msix_sync_irqs(struct xhci_hcd *xhci)
     391                 :            : {
     392         [ #  # ]:          0 :         struct usb_hcd *hcd = xhci_to_hcd(xhci);
     393                 :            : 
     394         [ #  # ]:          0 :         if (hcd->msix_enabled) {
     395                 :          0 :                 struct pci_dev *pdev = to_pci_dev(hcd->self.controller);
     396                 :          0 :                 int i;
     397                 :            : 
     398         [ #  # ]:          0 :                 for (i = 0; i < xhci->msix_count; i++)
     399                 :          0 :                         synchronize_irq(pci_irq_vector(pdev, i));
     400                 :            :         }
     401                 :          0 : }
     402                 :            : 
     403                 :          0 : static int xhci_try_enable_msi(struct usb_hcd *hcd)
     404                 :            : {
     405                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
     406                 :          0 :         struct pci_dev  *pdev;
     407                 :          0 :         int ret;
     408                 :            : 
     409                 :            :         /* The xhci platform device has set up IRQs through usb_add_hcd. */
     410         [ #  # ]:          0 :         if (xhci->quirks & XHCI_PLAT)
     411                 :            :                 return 0;
     412                 :            : 
     413         [ #  # ]:          0 :         pdev = to_pci_dev(xhci_to_hcd(xhci)->self.controller);
     414                 :            :         /*
     415                 :            :          * Some Fresco Logic host controllers advertise MSI, but fail to
     416                 :            :          * generate interrupts.  Don't even try to enable MSI.
     417                 :            :          */
     418         [ #  # ]:          0 :         if (xhci->quirks & XHCI_BROKEN_MSI)
     419                 :          0 :                 goto legacy_irq;
     420                 :            : 
     421                 :            :         /* unregister the legacy interrupt */
     422         [ #  # ]:          0 :         if (hcd->irq)
     423                 :          0 :                 free_irq(hcd->irq, hcd);
     424                 :          0 :         hcd->irq = 0;
     425                 :            : 
     426                 :          0 :         ret = xhci_setup_msix(xhci);
     427         [ #  # ]:          0 :         if (ret)
     428                 :            :                 /* fall back to msi*/
     429                 :          0 :                 ret = xhci_setup_msi(xhci);
     430                 :            : 
     431         [ #  # ]:          0 :         if (!ret) {
     432                 :          0 :                 hcd->msi_enabled = 1;
     433                 :          0 :                 return 0;
     434                 :            :         }
     435                 :            : 
     436         [ #  # ]:          0 :         if (!pdev->irq) {
     437                 :          0 :                 xhci_err(xhci, "No msi-x/msi found and no IRQ in BIOS\n");
     438                 :          0 :                 return -EINVAL;
     439                 :            :         }
     440                 :            : 
     441                 :          0 :  legacy_irq:
     442         [ #  # ]:          0 :         if (!strlen(hcd->irq_descr))
     443                 :          0 :                 snprintf(hcd->irq_descr, sizeof(hcd->irq_descr), "%s:usb%d",
     444                 :          0 :                          hcd->driver->description, hcd->self.busnum);
     445                 :            : 
     446                 :            :         /* fall back to legacy interrupt*/
     447                 :          0 :         ret = request_irq(pdev->irq, &usb_hcd_irq, IRQF_SHARED,
     448                 :          0 :                         hcd->irq_descr, hcd);
     449         [ #  # ]:          0 :         if (ret) {
     450                 :          0 :                 xhci_err(xhci, "request interrupt %d failed\n",
     451                 :            :                                 pdev->irq);
     452                 :          0 :                 return ret;
     453                 :            :         }
     454                 :          0 :         hcd->irq = pdev->irq;
     455                 :          0 :         return 0;
     456                 :            : }
     457                 :            : 
     458                 :            : #else
     459                 :            : 
     460                 :            : static inline int xhci_try_enable_msi(struct usb_hcd *hcd)
     461                 :            : {
     462                 :            :         return 0;
     463                 :            : }
     464                 :            : 
     465                 :            : static inline void xhci_cleanup_msix(struct xhci_hcd *xhci)
     466                 :            : {
     467                 :            : }
     468                 :            : 
     469                 :            : static inline void xhci_msix_sync_irqs(struct xhci_hcd *xhci)
     470                 :            : {
     471                 :            : }
     472                 :            : 
     473                 :            : #endif
     474                 :            : 
     475                 :          0 : static void compliance_mode_recovery(struct timer_list *t)
     476                 :            : {
     477                 :          0 :         struct xhci_hcd *xhci;
     478                 :          0 :         struct usb_hcd *hcd;
     479                 :          0 :         struct xhci_hub *rhub;
     480                 :          0 :         u32 temp;
     481                 :          0 :         int i;
     482                 :            : 
     483                 :          0 :         xhci = from_timer(xhci, t, comp_mode_recovery_timer);
     484                 :          0 :         rhub = &xhci->usb3_rhub;
     485                 :            : 
     486         [ #  # ]:          0 :         for (i = 0; i < rhub->num_ports; i++) {
     487                 :          0 :                 temp = readl(rhub->ports[i]->addr);
     488         [ #  # ]:          0 :                 if ((temp & PORT_PLS_MASK) == USB_SS_PORT_LS_COMP_MOD) {
     489                 :            :                         /*
     490                 :            :                          * Compliance Mode Detected. Letting USB Core
     491                 :            :                          * handle the Warm Reset
     492                 :            :                          */
     493                 :          0 :                         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
     494                 :            :                                         "Compliance mode detected->port %d",
     495                 :            :                                         i + 1);
     496                 :          0 :                         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
     497                 :            :                                         "Attempting compliance mode recovery");
     498                 :          0 :                         hcd = xhci->shared_hcd;
     499                 :            : 
     500         [ #  # ]:          0 :                         if (hcd->state == HC_STATE_SUSPENDED)
     501                 :          0 :                                 usb_hcd_resume_root_hub(hcd);
     502                 :            : 
     503                 :          0 :                         usb_hcd_poll_rh_status(hcd);
     504                 :            :                 }
     505                 :            :         }
     506                 :            : 
     507         [ #  # ]:          0 :         if (xhci->port_status_u0 != ((1 << rhub->num_ports) - 1))
     508                 :          0 :                 mod_timer(&xhci->comp_mode_recovery_timer,
     509                 :            :                         jiffies + msecs_to_jiffies(COMP_MODE_RCVRY_MSECS));
     510                 :          0 : }
     511                 :            : 
     512                 :            : /*
     513                 :            :  * Quirk to work around issue generated by the SN65LVPE502CP USB3.0 re-driver
     514                 :            :  * that causes ports behind that hardware to enter compliance mode sometimes.
     515                 :            :  * The quirk creates a timer that polls every 2 seconds the link state of
     516                 :            :  * each host controller's port and recovers it by issuing a Warm reset
     517                 :            :  * if Compliance mode is detected, otherwise the port will become "dead" (no
     518                 :            :  * device connections or disconnections will be detected anymore). Becasue no
     519                 :            :  * status event is generated when entering compliance mode (per xhci spec),
     520                 :            :  * this quirk is needed on systems that have the failing hardware installed.
     521                 :            :  */
     522                 :          0 : static void compliance_mode_recovery_timer_init(struct xhci_hcd *xhci)
     523                 :            : {
     524                 :          0 :         xhci->port_status_u0 = 0;
     525                 :          0 :         timer_setup(&xhci->comp_mode_recovery_timer, compliance_mode_recovery,
     526                 :            :                     0);
     527                 :          0 :         xhci->comp_mode_recovery_timer.expires = jiffies +
     528                 :            :                         msecs_to_jiffies(COMP_MODE_RCVRY_MSECS);
     529                 :            : 
     530                 :          0 :         add_timer(&xhci->comp_mode_recovery_timer);
     531                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
     532                 :            :                         "Compliance mode recovery timer initialized");
     533                 :          0 : }
     534                 :            : 
     535                 :            : /*
     536                 :            :  * This function identifies the systems that have installed the SN65LVPE502CP
     537                 :            :  * USB3.0 re-driver and that need the Compliance Mode Quirk.
     538                 :            :  * Systems:
     539                 :            :  * Vendor: Hewlett-Packard -> System Models: Z420, Z620 and Z820
     540                 :            :  */
     541                 :          0 : static bool xhci_compliance_mode_recovery_timer_quirk_check(void)
     542                 :            : {
     543                 :          0 :         const char *dmi_product_name, *dmi_sys_vendor;
     544                 :            : 
     545                 :          0 :         dmi_product_name = dmi_get_system_info(DMI_PRODUCT_NAME);
     546                 :          0 :         dmi_sys_vendor = dmi_get_system_info(DMI_SYS_VENDOR);
     547         [ #  # ]:          0 :         if (!dmi_product_name || !dmi_sys_vendor)
     548                 :            :                 return false;
     549                 :            : 
     550         [ #  # ]:          0 :         if (!(strstr(dmi_sys_vendor, "Hewlett-Packard")))
     551                 :            :                 return false;
     552                 :            : 
     553         [ #  # ]:          0 :         if (strstr(dmi_product_name, "Z420") ||
     554         [ #  # ]:          0 :                         strstr(dmi_product_name, "Z620") ||
     555         [ #  # ]:          0 :                         strstr(dmi_product_name, "Z820") ||
     556         [ #  # ]:          0 :                         strstr(dmi_product_name, "Z1 Workstation"))
     557                 :          0 :                 return true;
     558                 :            : 
     559                 :            :         return false;
     560                 :            : }
     561                 :            : 
     562                 :          0 : static int xhci_all_ports_seen_u0(struct xhci_hcd *xhci)
     563                 :            : {
     564                 :          0 :         return (xhci->port_status_u0 == ((1 << xhci->usb3_rhub.num_ports) - 1));
     565                 :            : }
     566                 :            : 
     567                 :            : 
     568                 :            : /*
     569                 :            :  * Initialize memory for HCD and xHC (one-time init).
     570                 :            :  *
     571                 :            :  * Program the PAGESIZE register, initialize the device context array, create
     572                 :            :  * device contexts (?), set up a command ring segment (or two?), create event
     573                 :            :  * ring (one for now).
     574                 :            :  */
     575                 :          0 : static int xhci_init(struct usb_hcd *hcd)
     576                 :            : {
     577                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
     578                 :          0 :         int retval = 0;
     579                 :            : 
     580                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_init");
     581         [ #  # ]:          0 :         spin_lock_init(&xhci->lock);
     582   [ #  #  #  # ]:          0 :         if (xhci->hci_version == 0x95 && link_quirk) {
     583                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
     584                 :            :                                 "QUIRK: Not clearing Link TRB chain bits.");
     585                 :          0 :                 xhci->quirks |= XHCI_LINK_TRB_QUIRK;
     586                 :            :         } else {
     587                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     588                 :            :                                 "xHCI doesn't need link TRB QUIRK");
     589                 :            :         }
     590                 :          0 :         retval = xhci_mem_init(xhci, GFP_KERNEL);
     591                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "Finished xhci_init");
     592                 :            : 
     593                 :            :         /* Initializing Compliance Mode Recovery Data If Needed */
     594         [ #  # ]:          0 :         if (xhci_compliance_mode_recovery_timer_quirk_check()) {
     595                 :          0 :                 xhci->quirks |= XHCI_COMP_MODE_QUIRK;
     596                 :          0 :                 compliance_mode_recovery_timer_init(xhci);
     597                 :            :         }
     598                 :            : 
     599                 :          0 :         return retval;
     600                 :            : }
     601                 :            : 
     602                 :            : /*-------------------------------------------------------------------------*/
     603                 :            : 
     604                 :            : 
     605                 :          0 : static int xhci_run_finished(struct xhci_hcd *xhci)
     606                 :            : {
     607         [ #  # ]:          0 :         if (xhci_start(xhci)) {
     608                 :          0 :                 xhci_halt(xhci);
     609                 :          0 :                 return -ENODEV;
     610                 :            :         }
     611                 :          0 :         xhci->shared_hcd->state = HC_STATE_RUNNING;
     612                 :          0 :         xhci->cmd_ring_state = CMD_RING_STATE_RUNNING;
     613                 :            : 
     614         [ #  # ]:          0 :         if (xhci->quirks & XHCI_NEC_HOST)
     615                 :          0 :                 xhci_ring_cmd_db(xhci);
     616                 :            : 
     617                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     618                 :            :                         "Finished xhci_run for USB3 roothub");
     619                 :          0 :         return 0;
     620                 :            : }
     621                 :            : 
     622                 :            : /*
     623                 :            :  * Start the HC after it was halted.
     624                 :            :  *
     625                 :            :  * This function is called by the USB core when the HC driver is added.
     626                 :            :  * Its opposite is xhci_stop().
     627                 :            :  *
     628                 :            :  * xhci_init() must be called once before this function can be called.
     629                 :            :  * Reset the HC, enable device slot contexts, program DCBAAP, and
     630                 :            :  * set command ring pointer and event ring pointer.
     631                 :            :  *
     632                 :            :  * Setup MSI-X vectors and enable interrupts.
     633                 :            :  */
     634                 :          0 : int xhci_run(struct usb_hcd *hcd)
     635                 :            : {
     636                 :          0 :         u32 temp;
     637                 :          0 :         u64 temp_64;
     638                 :          0 :         int ret;
     639                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
     640                 :            : 
     641                 :            :         /* Start the xHCI host controller running only after the USB 2.0 roothub
     642                 :            :          * is setup.
     643                 :            :          */
     644                 :            : 
     645                 :          0 :         hcd->uses_new_polling = 1;
     646         [ #  # ]:          0 :         if (!usb_hcd_is_primary_hcd(hcd))
     647                 :          0 :                 return xhci_run_finished(xhci);
     648                 :            : 
     649                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "xhci_run");
     650                 :            : 
     651                 :          0 :         ret = xhci_try_enable_msi(hcd);
     652         [ #  # ]:          0 :         if (ret)
     653                 :            :                 return ret;
     654                 :            : 
     655                 :          0 :         temp_64 = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
     656                 :          0 :         temp_64 &= ~ERST_PTR_MASK;
     657                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     658                 :            :                         "ERST deq = 64'h%0lx", (long unsigned int) temp_64);
     659                 :            : 
     660                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     661                 :            :                         "// Set the interrupt modulation register");
     662                 :          0 :         temp = readl(&xhci->ir_set->irq_control);
     663                 :          0 :         temp &= ~ER_IRQ_INTERVAL_MASK;
     664                 :          0 :         temp |= (xhci->imod_interval / 250) & ER_IRQ_INTERVAL_MASK;
     665                 :          0 :         writel(temp, &xhci->ir_set->irq_control);
     666                 :            : 
     667                 :            :         /* Set the HCD state before we enable the irqs */
     668                 :          0 :         temp = readl(&xhci->op_regs->command);
     669                 :          0 :         temp |= (CMD_EIE);
     670                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     671                 :            :                         "// Enable interrupts, cmd = 0x%x.", temp);
     672                 :          0 :         writel(temp, &xhci->op_regs->command);
     673                 :            : 
     674                 :          0 :         temp = readl(&xhci->ir_set->irq_pending);
     675                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     676                 :            :                         "// Enabling event ring interrupter %p by writing 0x%x to irq_pending",
     677                 :          0 :                         xhci->ir_set, (unsigned int) ER_IRQ_ENABLE(temp));
     678                 :          0 :         writel(ER_IRQ_ENABLE(temp), &xhci->ir_set->irq_pending);
     679                 :            : 
     680         [ #  # ]:          0 :         if (xhci->quirks & XHCI_NEC_HOST) {
     681                 :          0 :                 struct xhci_command *command;
     682                 :            : 
     683                 :          0 :                 command = xhci_alloc_command(xhci, false, GFP_KERNEL);
     684         [ #  # ]:          0 :                 if (!command)
     685                 :            :                         return -ENOMEM;
     686                 :            : 
     687                 :          0 :                 ret = xhci_queue_vendor_command(xhci, command, 0, 0, 0,
     688                 :            :                                 TRB_TYPE(TRB_NEC_GET_FW));
     689         [ #  # ]:          0 :                 if (ret)
     690                 :          0 :                         xhci_free_command(xhci, command);
     691                 :            :         }
     692                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     693                 :            :                         "Finished xhci_run for USB2 roothub");
     694                 :            : 
     695                 :          0 :         xhci_dbc_init(xhci);
     696                 :            : 
     697                 :          0 :         xhci_debugfs_init(xhci);
     698                 :            : 
     699                 :          0 :         return 0;
     700                 :            : }
     701                 :            : EXPORT_SYMBOL_GPL(xhci_run);
     702                 :            : 
     703                 :            : /*
     704                 :            :  * Stop xHCI driver.
     705                 :            :  *
     706                 :            :  * This function is called by the USB core when the HC driver is removed.
     707                 :            :  * Its opposite is xhci_run().
     708                 :            :  *
     709                 :            :  * Disable device contexts, disable IRQs, and quiesce the HC.
     710                 :            :  * Reset the HC, finish any completed transactions, and cleanup memory.
     711                 :            :  */
     712                 :          0 : static void xhci_stop(struct usb_hcd *hcd)
     713                 :            : {
     714                 :          0 :         u32 temp;
     715                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
     716                 :            : 
     717                 :          0 :         mutex_lock(&xhci->mutex);
     718                 :            : 
     719                 :            :         /* Only halt host and free memory after both hcds are removed */
     720         [ #  # ]:          0 :         if (!usb_hcd_is_primary_hcd(hcd)) {
     721                 :          0 :                 mutex_unlock(&xhci->mutex);
     722                 :          0 :                 return;
     723                 :            :         }
     724                 :            : 
     725                 :          0 :         xhci_dbc_exit(xhci);
     726                 :            : 
     727                 :          0 :         spin_lock_irq(&xhci->lock);
     728                 :          0 :         xhci->xhc_state |= XHCI_STATE_HALTED;
     729                 :          0 :         xhci->cmd_ring_state = CMD_RING_STATE_STOPPED;
     730                 :          0 :         xhci_halt(xhci);
     731                 :          0 :         xhci_reset(xhci);
     732                 :          0 :         spin_unlock_irq(&xhci->lock);
     733                 :            : 
     734                 :          0 :         xhci_cleanup_msix(xhci);
     735                 :            : 
     736                 :            :         /* Deleting Compliance Mode Recovery Timer */
     737         [ #  # ]:          0 :         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
     738         [ #  # ]:          0 :                         (!(xhci_all_ports_seen_u0(xhci)))) {
     739                 :          0 :                 del_timer_sync(&xhci->comp_mode_recovery_timer);
     740                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
     741                 :            :                                 "%s: compliance mode recovery timer deleted",
     742                 :            :                                 __func__);
     743                 :            :         }
     744                 :            : 
     745         [ #  # ]:          0 :         if (xhci->quirks & XHCI_AMD_PLL_FIX)
     746                 :          0 :                 usb_amd_dev_put();
     747                 :            : 
     748                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     749                 :            :                         "// Disabling event ring interrupts");
     750                 :          0 :         temp = readl(&xhci->op_regs->status);
     751                 :          0 :         writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
     752                 :          0 :         temp = readl(&xhci->ir_set->irq_pending);
     753                 :          0 :         writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
     754                 :            : 
     755                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init, "cleaning up memory");
     756                 :          0 :         xhci_mem_cleanup(xhci);
     757                 :          0 :         xhci_debugfs_exit(xhci);
     758                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     759                 :            :                         "xhci_stop completed - status = %x",
     760                 :          0 :                         readl(&xhci->op_regs->status));
     761                 :          0 :         mutex_unlock(&xhci->mutex);
     762                 :            : }
     763                 :            : 
     764                 :            : /*
     765                 :            :  * Shutdown HC (not bus-specific)
     766                 :            :  *
     767                 :            :  * This is called when the machine is rebooting or halting.  We assume that the
     768                 :            :  * machine will be powered off, and the HC's internal state will be reset.
     769                 :            :  * Don't bother to free memory.
     770                 :            :  *
     771                 :            :  * This will only ever be called with the main usb_hcd (the USB3 roothub).
     772                 :            :  */
     773                 :          0 : void xhci_shutdown(struct usb_hcd *hcd)
     774                 :            : {
     775                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
     776                 :            : 
     777         [ #  # ]:          0 :         if (xhci->quirks & XHCI_SPURIOUS_REBOOT)
     778                 :          0 :                 usb_disable_xhci_ports(to_pci_dev(hcd->self.sysdev));
     779                 :            : 
     780                 :          0 :         spin_lock_irq(&xhci->lock);
     781                 :          0 :         xhci_halt(xhci);
     782                 :            :         /* Workaround for spurious wakeups at shutdown with HSW */
     783         [ #  # ]:          0 :         if (xhci->quirks & XHCI_SPURIOUS_WAKEUP)
     784                 :          0 :                 xhci_reset(xhci);
     785                 :          0 :         spin_unlock_irq(&xhci->lock);
     786                 :            : 
     787                 :          0 :         xhci_cleanup_msix(xhci);
     788                 :            : 
     789                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     790                 :            :                         "xhci_shutdown completed - status = %x",
     791                 :          0 :                         readl(&xhci->op_regs->status));
     792                 :          0 : }
     793                 :            : EXPORT_SYMBOL_GPL(xhci_shutdown);
     794                 :            : 
     795                 :            : #ifdef CONFIG_PM
     796                 :          0 : static void xhci_save_registers(struct xhci_hcd *xhci)
     797                 :            : {
     798                 :          0 :         xhci->s3.command = readl(&xhci->op_regs->command);
     799                 :          0 :         xhci->s3.dev_nt = readl(&xhci->op_regs->dev_notification);
     800                 :          0 :         xhci->s3.dcbaa_ptr = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
     801                 :          0 :         xhci->s3.config_reg = readl(&xhci->op_regs->config_reg);
     802                 :          0 :         xhci->s3.erst_size = readl(&xhci->ir_set->erst_size);
     803                 :          0 :         xhci->s3.erst_base = xhci_read_64(xhci, &xhci->ir_set->erst_base);
     804                 :          0 :         xhci->s3.erst_dequeue = xhci_read_64(xhci, &xhci->ir_set->erst_dequeue);
     805                 :          0 :         xhci->s3.irq_pending = readl(&xhci->ir_set->irq_pending);
     806                 :          0 :         xhci->s3.irq_control = readl(&xhci->ir_set->irq_control);
     807                 :          0 : }
     808                 :            : 
     809                 :          0 : static void xhci_restore_registers(struct xhci_hcd *xhci)
     810                 :            : {
     811                 :          0 :         writel(xhci->s3.command, &xhci->op_regs->command);
     812                 :          0 :         writel(xhci->s3.dev_nt, &xhci->op_regs->dev_notification);
     813                 :          0 :         xhci_write_64(xhci, xhci->s3.dcbaa_ptr, &xhci->op_regs->dcbaa_ptr);
     814                 :          0 :         writel(xhci->s3.config_reg, &xhci->op_regs->config_reg);
     815                 :          0 :         writel(xhci->s3.erst_size, &xhci->ir_set->erst_size);
     816                 :          0 :         xhci_write_64(xhci, xhci->s3.erst_base, &xhci->ir_set->erst_base);
     817                 :          0 :         xhci_write_64(xhci, xhci->s3.erst_dequeue, &xhci->ir_set->erst_dequeue);
     818                 :          0 :         writel(xhci->s3.irq_pending, &xhci->ir_set->irq_pending);
     819                 :          0 :         writel(xhci->s3.irq_control, &xhci->ir_set->irq_control);
     820                 :          0 : }
     821                 :            : 
     822                 :          0 : static void xhci_set_cmd_ring_deq(struct xhci_hcd *xhci)
     823                 :            : {
     824                 :          0 :         u64     val_64;
     825                 :            : 
     826                 :            :         /* step 2: initialize command ring buffer */
     827                 :          0 :         val_64 = xhci_read_64(xhci, &xhci->op_regs->cmd_ring);
     828                 :          0 :         val_64 = (val_64 & (u64) CMD_RING_RSVD_BITS) |
     829                 :          0 :                 (xhci_trb_virt_to_dma(xhci->cmd_ring->deq_seg,
     830                 :          0 :                                       xhci->cmd_ring->dequeue) &
     831                 :            :                  (u64) ~CMD_RING_RSVD_BITS) |
     832                 :          0 :                 xhci->cmd_ring->cycle_state;
     833                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_init,
     834                 :            :                         "// Setting command ring address to 0x%llx",
     835                 :            :                         (long unsigned long) val_64);
     836                 :          0 :         xhci_write_64(xhci, val_64, &xhci->op_regs->cmd_ring);
     837                 :          0 : }
     838                 :            : 
     839                 :            : /*
     840                 :            :  * The whole command ring must be cleared to zero when we suspend the host.
     841                 :            :  *
     842                 :            :  * The host doesn't save the command ring pointer in the suspend well, so we
     843                 :            :  * need to re-program it on resume.  Unfortunately, the pointer must be 64-byte
     844                 :            :  * aligned, because of the reserved bits in the command ring dequeue pointer
     845                 :            :  * register.  Therefore, we can't just set the dequeue pointer back in the
     846                 :            :  * middle of the ring (TRBs are 16-byte aligned).
     847                 :            :  */
     848                 :          0 : static void xhci_clear_command_ring(struct xhci_hcd *xhci)
     849                 :            : {
     850                 :          0 :         struct xhci_ring *ring;
     851                 :          0 :         struct xhci_segment *seg;
     852                 :            : 
     853                 :          0 :         ring = xhci->cmd_ring;
     854                 :          0 :         seg = ring->deq_seg;
     855                 :          0 :         do {
     856                 :          0 :                 memset(seg->trbs, 0,
     857                 :            :                         sizeof(union xhci_trb) * (TRBS_PER_SEGMENT - 1));
     858                 :          0 :                 seg->trbs[TRBS_PER_SEGMENT - 1].link.control &=
     859                 :            :                         cpu_to_le32(~TRB_CYCLE);
     860                 :          0 :                 seg = seg->next;
     861         [ #  # ]:          0 :         } while (seg != ring->deq_seg);
     862                 :            : 
     863                 :            :         /* Reset the software enqueue and dequeue pointers */
     864                 :          0 :         ring->deq_seg = ring->first_seg;
     865                 :          0 :         ring->dequeue = ring->first_seg->trbs;
     866                 :          0 :         ring->enq_seg = ring->deq_seg;
     867                 :          0 :         ring->enqueue = ring->dequeue;
     868                 :            : 
     869                 :          0 :         ring->num_trbs_free = ring->num_segs * (TRBS_PER_SEGMENT - 1) - 1;
     870                 :            :         /*
     871                 :            :          * Ring is now zeroed, so the HW should look for change of ownership
     872                 :            :          * when the cycle bit is set to 1.
     873                 :            :          */
     874                 :          0 :         ring->cycle_state = 1;
     875                 :            : 
     876                 :            :         /*
     877                 :            :          * Reset the hardware dequeue pointer.
     878                 :            :          * Yes, this will need to be re-written after resume, but we're paranoid
     879                 :            :          * and want to make sure the hardware doesn't access bogus memory
     880                 :            :          * because, say, the BIOS or an SMI started the host without changing
     881                 :            :          * the command ring pointers.
     882                 :            :          */
     883                 :          0 :         xhci_set_cmd_ring_deq(xhci);
     884                 :          0 : }
     885                 :            : 
     886                 :          0 : static void xhci_disable_port_wake_on_bits(struct xhci_hcd *xhci)
     887                 :            : {
     888                 :          0 :         struct xhci_port **ports;
     889                 :          0 :         int port_index;
     890                 :          0 :         unsigned long flags;
     891                 :          0 :         u32 t1, t2, portsc;
     892                 :            : 
     893                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
     894                 :            : 
     895                 :            :         /* disable usb3 ports Wake bits */
     896                 :          0 :         port_index = xhci->usb3_rhub.num_ports;
     897                 :          0 :         ports = xhci->usb3_rhub.ports;
     898         [ #  # ]:          0 :         while (port_index--) {
     899                 :          0 :                 t1 = readl(ports[port_index]->addr);
     900                 :          0 :                 portsc = t1;
     901                 :          0 :                 t1 = xhci_port_state_to_neutral(t1);
     902                 :          0 :                 t2 = t1 & ~PORT_WAKE_BITS;
     903         [ #  # ]:          0 :                 if (t1 != t2) {
     904                 :          0 :                         writel(t2, ports[port_index]->addr);
     905                 :            :                         xhci_dbg(xhci, "disable wake bits port %d-%d, portsc: 0x%x, write: 0x%x\n",
     906                 :            :                                  xhci->usb3_rhub.hcd->self.busnum,
     907                 :            :                                  port_index + 1, portsc, t2);
     908                 :            :                 }
     909                 :            :         }
     910                 :            : 
     911                 :            :         /* disable usb2 ports Wake bits */
     912                 :          0 :         port_index = xhci->usb2_rhub.num_ports;
     913                 :          0 :         ports = xhci->usb2_rhub.ports;
     914         [ #  # ]:          0 :         while (port_index--) {
     915                 :          0 :                 t1 = readl(ports[port_index]->addr);
     916                 :          0 :                 portsc = t1;
     917                 :          0 :                 t1 = xhci_port_state_to_neutral(t1);
     918                 :          0 :                 t2 = t1 & ~PORT_WAKE_BITS;
     919         [ #  # ]:          0 :                 if (t1 != t2) {
     920                 :          0 :                         writel(t2, ports[port_index]->addr);
     921                 :            :                         xhci_dbg(xhci, "disable wake bits port %d-%d, portsc: 0x%x, write: 0x%x\n",
     922                 :            :                                  xhci->usb2_rhub.hcd->self.busnum,
     923                 :            :                                  port_index + 1, portsc, t2);
     924                 :            :                 }
     925                 :            :         }
     926                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
     927                 :          0 : }
     928                 :            : 
     929                 :          0 : static bool xhci_pending_portevent(struct xhci_hcd *xhci)
     930                 :            : {
     931                 :          0 :         struct xhci_port        **ports;
     932                 :          0 :         int                     port_index;
     933                 :          0 :         u32                     status;
     934                 :          0 :         u32                     portsc;
     935                 :            : 
     936                 :          0 :         status = readl(&xhci->op_regs->status);
     937         [ #  # ]:          0 :         if (status & STS_EINT)
     938                 :            :                 return true;
     939                 :            :         /*
     940                 :            :          * Checking STS_EINT is not enough as there is a lag between a change
     941                 :            :          * bit being set and the Port Status Change Event that it generated
     942                 :            :          * being written to the Event Ring. See note in xhci 1.1 section 4.19.2.
     943                 :            :          */
     944                 :            : 
     945                 :          0 :         port_index = xhci->usb2_rhub.num_ports;
     946                 :          0 :         ports = xhci->usb2_rhub.ports;
     947         [ #  # ]:          0 :         while (port_index--) {
     948                 :          0 :                 portsc = readl(ports[port_index]->addr);
     949         [ #  # ]:          0 :                 if (portsc & PORT_CHANGE_MASK ||
     950         [ #  # ]:          0 :                     (portsc & PORT_PLS_MASK) == XDEV_RESUME)
     951                 :            :                         return true;
     952                 :            :         }
     953                 :          0 :         port_index = xhci->usb3_rhub.num_ports;
     954                 :          0 :         ports = xhci->usb3_rhub.ports;
     955         [ #  # ]:          0 :         while (port_index--) {
     956                 :          0 :                 portsc = readl(ports[port_index]->addr);
     957         [ #  # ]:          0 :                 if (portsc & PORT_CHANGE_MASK ||
     958         [ #  # ]:          0 :                     (portsc & PORT_PLS_MASK) == XDEV_RESUME)
     959                 :            :                         return true;
     960                 :            :         }
     961                 :            :         return false;
     962                 :            : }
     963                 :            : 
     964                 :            : /*
     965                 :            :  * Stop HC (not bus-specific)
     966                 :            :  *
     967                 :            :  * This is called when the machine transition into S3/S4 mode.
     968                 :            :  *
     969                 :            :  */
     970                 :          0 : int xhci_suspend(struct xhci_hcd *xhci, bool do_wakeup)
     971                 :            : {
     972                 :          0 :         int                     rc = 0;
     973                 :          0 :         unsigned int            delay = XHCI_MAX_HALT_USEC * 2;
     974         [ #  # ]:          0 :         struct usb_hcd          *hcd = xhci_to_hcd(xhci);
     975                 :          0 :         u32                     command;
     976                 :          0 :         u32                     res;
     977                 :            : 
     978         [ #  # ]:          0 :         if (!hcd->state)
     979                 :            :                 return 0;
     980                 :            : 
     981         [ #  # ]:          0 :         if (hcd->state != HC_STATE_SUSPENDED ||
     982         [ #  # ]:          0 :                         xhci->shared_hcd->state != HC_STATE_SUSPENDED)
     983                 :            :                 return -EINVAL;
     984                 :            : 
     985         [ #  # ]:          0 :         xhci_dbc_suspend(xhci);
     986                 :            : 
     987                 :            :         /* Clear root port wake on bits if wakeup not allowed. */
     988         [ #  # ]:          0 :         if (!do_wakeup)
     989                 :          0 :                 xhci_disable_port_wake_on_bits(xhci);
     990                 :            : 
     991                 :            :         /* Don't poll the roothubs on bus suspend. */
     992                 :          0 :         xhci_dbg(xhci, "%s: stopping port polling.\n", __func__);
     993                 :          0 :         clear_bit(HCD_FLAG_POLL_RH, &hcd->flags);
     994                 :          0 :         del_timer_sync(&hcd->rh_timer);
     995                 :          0 :         clear_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
     996                 :          0 :         del_timer_sync(&xhci->shared_hcd->rh_timer);
     997                 :            : 
     998         [ #  # ]:          0 :         if (xhci->quirks & XHCI_SUSPEND_DELAY)
     999                 :          0 :                 usleep_range(1000, 1500);
    1000                 :            : 
    1001                 :          0 :         spin_lock_irq(&xhci->lock);
    1002                 :          0 :         clear_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
    1003                 :          0 :         clear_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
    1004                 :            :         /* step 1: stop endpoint */
    1005                 :            :         /* skipped assuming that port suspend has done */
    1006                 :            : 
    1007                 :            :         /* step 2: clear Run/Stop bit */
    1008                 :          0 :         command = readl(&xhci->op_regs->command);
    1009                 :          0 :         command &= ~CMD_RUN;
    1010                 :          0 :         writel(command, &xhci->op_regs->command);
    1011                 :            : 
    1012                 :            :         /* Some chips from Fresco Logic need an extraordinary delay */
    1013         [ #  # ]:          0 :         delay *= (xhci->quirks & XHCI_SLOW_SUSPEND) ? 10 : 1;
    1014                 :            : 
    1015         [ #  # ]:          0 :         if (xhci_handshake(&xhci->op_regs->status,
    1016                 :            :                       STS_HALT, STS_HALT, delay)) {
    1017                 :          0 :                 xhci_warn(xhci, "WARN: xHC CMD_RUN timeout\n");
    1018                 :          0 :                 spin_unlock_irq(&xhci->lock);
    1019                 :          0 :                 return -ETIMEDOUT;
    1020                 :            :         }
    1021                 :          0 :         xhci_clear_command_ring(xhci);
    1022                 :            : 
    1023                 :            :         /* step 3: save registers */
    1024                 :          0 :         xhci_save_registers(xhci);
    1025                 :            : 
    1026                 :            :         /* step 4: set CSS flag */
    1027                 :          0 :         command = readl(&xhci->op_regs->command);
    1028                 :          0 :         command |= CMD_CSS;
    1029                 :          0 :         writel(command, &xhci->op_regs->command);
    1030                 :          0 :         xhci->broken_suspend = 0;
    1031         [ #  # ]:          0 :         if (xhci_handshake(&xhci->op_regs->status,
    1032                 :            :                                 STS_SAVE, 0, 20 * 1000)) {
    1033                 :            :         /*
    1034                 :            :          * AMD SNPS xHC 3.0 occasionally does not clear the
    1035                 :            :          * SSS bit of USBSTS and when driver tries to poll
    1036                 :            :          * to see if the xHC clears BIT(8) which never happens
    1037                 :            :          * and driver assumes that controller is not responding
    1038                 :            :          * and times out. To workaround this, its good to check
    1039                 :            :          * if SRE and HCE bits are not set (as per xhci
    1040                 :            :          * Section 5.4.2) and bypass the timeout.
    1041                 :            :          */
    1042                 :          0 :                 res = readl(&xhci->op_regs->status);
    1043         [ #  # ]:          0 :                 if ((xhci->quirks & XHCI_SNPS_BROKEN_SUSPEND) &&
    1044         [ #  # ]:          0 :                     (((res & STS_SRE) == 0) &&
    1045                 :            :                                 ((res & STS_HCE) == 0))) {
    1046                 :          0 :                         xhci->broken_suspend = 1;
    1047                 :            :                 } else {
    1048                 :          0 :                         xhci_warn(xhci, "WARN: xHC save state timeout\n");
    1049                 :          0 :                         spin_unlock_irq(&xhci->lock);
    1050                 :          0 :                         return -ETIMEDOUT;
    1051                 :            :                 }
    1052                 :            :         }
    1053                 :          0 :         spin_unlock_irq(&xhci->lock);
    1054                 :            : 
    1055                 :            :         /*
    1056                 :            :          * Deleting Compliance Mode Recovery Timer because the xHCI Host
    1057                 :            :          * is about to be suspended.
    1058                 :            :          */
    1059         [ #  # ]:          0 :         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
    1060         [ #  # ]:          0 :                         (!(xhci_all_ports_seen_u0(xhci)))) {
    1061                 :          0 :                 del_timer_sync(&xhci->comp_mode_recovery_timer);
    1062                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    1063                 :            :                                 "%s: compliance mode recovery timer deleted",
    1064                 :            :                                 __func__);
    1065                 :            :         }
    1066                 :            : 
    1067                 :            :         /* step 5: remove core well power */
    1068                 :            :         /* synchronize irq when using MSI-X */
    1069                 :          0 :         xhci_msix_sync_irqs(xhci);
    1070                 :            : 
    1071                 :          0 :         return rc;
    1072                 :            : }
    1073                 :            : EXPORT_SYMBOL_GPL(xhci_suspend);
    1074                 :            : 
    1075                 :            : /*
    1076                 :            :  * start xHC (not bus-specific)
    1077                 :            :  *
    1078                 :            :  * This is called when the machine transition from S3/S4 mode.
    1079                 :            :  *
    1080                 :            :  */
    1081                 :          0 : int xhci_resume(struct xhci_hcd *xhci, bool hibernated)
    1082                 :            : {
    1083                 :          0 :         u32                     command, temp = 0;
    1084         [ #  # ]:          0 :         struct usb_hcd          *hcd = xhci_to_hcd(xhci);
    1085                 :          0 :         struct usb_hcd          *secondary_hcd;
    1086                 :          0 :         int                     retval = 0;
    1087                 :          0 :         bool                    comp_timer_running = false;
    1088                 :            : 
    1089         [ #  # ]:          0 :         if (!hcd->state)
    1090                 :            :                 return 0;
    1091                 :            : 
    1092                 :            :         /* Wait a bit if either of the roothubs need to settle from the
    1093                 :            :          * transition into bus suspend.
    1094                 :            :          */
    1095                 :            : 
    1096         [ #  # ]:          0 :         if (time_before(jiffies, xhci->usb2_rhub.bus_state.next_statechange) ||
    1097         [ #  # ]:          0 :             time_before(jiffies, xhci->usb3_rhub.bus_state.next_statechange))
    1098                 :          0 :                 msleep(100);
    1099                 :            : 
    1100                 :          0 :         set_bit(HCD_FLAG_HW_ACCESSIBLE, &hcd->flags);
    1101                 :          0 :         set_bit(HCD_FLAG_HW_ACCESSIBLE, &xhci->shared_hcd->flags);
    1102                 :            : 
    1103                 :          0 :         spin_lock_irq(&xhci->lock);
    1104   [ #  #  #  # ]:          0 :         if ((xhci->quirks & XHCI_RESET_ON_RESUME) || xhci->broken_suspend)
    1105                 :            :                 hibernated = true;
    1106                 :            : 
    1107         [ #  # ]:          0 :         if (!hibernated) {
    1108                 :            :                 /*
    1109                 :            :                  * Some controllers might lose power during suspend, so wait
    1110                 :            :                  * for controller not ready bit to clear, just as in xHC init.
    1111                 :            :                  */
    1112                 :          0 :                 retval = xhci_handshake(&xhci->op_regs->status,
    1113                 :            :                                         STS_CNR, 0, 10 * 1000 * 1000);
    1114         [ #  # ]:          0 :                 if (retval) {
    1115                 :          0 :                         xhci_warn(xhci, "Controller not ready at resume %d\n",
    1116                 :            :                                   retval);
    1117                 :          0 :                         spin_unlock_irq(&xhci->lock);
    1118                 :          0 :                         return retval;
    1119                 :            :                 }
    1120                 :            :                 /* step 1: restore register */
    1121                 :          0 :                 xhci_restore_registers(xhci);
    1122                 :            :                 /* step 2: initialize command ring buffer */
    1123                 :          0 :                 xhci_set_cmd_ring_deq(xhci);
    1124                 :            :                 /* step 3: restore state and start state*/
    1125                 :            :                 /* step 3: set CRS flag */
    1126                 :          0 :                 command = readl(&xhci->op_regs->command);
    1127                 :          0 :                 command |= CMD_CRS;
    1128                 :          0 :                 writel(command, &xhci->op_regs->command);
    1129                 :            :                 /*
    1130                 :            :                  * Some controllers take up to 55+ ms to complete the controller
    1131                 :            :                  * restore so setting the timeout to 100ms. Xhci specification
    1132                 :            :                  * doesn't mention any timeout value.
    1133                 :            :                  */
    1134         [ #  # ]:          0 :                 if (xhci_handshake(&xhci->op_regs->status,
    1135                 :            :                               STS_RESTORE, 0, 100 * 1000)) {
    1136                 :          0 :                         xhci_warn(xhci, "WARN: xHC restore state timeout\n");
    1137                 :          0 :                         spin_unlock_irq(&xhci->lock);
    1138                 :          0 :                         return -ETIMEDOUT;
    1139                 :            :                 }
    1140                 :          0 :                 temp = readl(&xhci->op_regs->status);
    1141                 :            :         }
    1142                 :            : 
    1143                 :            :         /* If restore operation fails, re-initialize the HC during resume */
    1144   [ #  #  #  # ]:          0 :         if ((temp & STS_SRE) || hibernated) {
    1145                 :            : 
    1146         [ #  # ]:          0 :                 if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) &&
    1147         [ #  # ]:          0 :                                 !(xhci_all_ports_seen_u0(xhci))) {
    1148                 :          0 :                         del_timer_sync(&xhci->comp_mode_recovery_timer);
    1149                 :          0 :                         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    1150                 :            :                                 "Compliance Mode Recovery Timer deleted!");
    1151                 :            :                 }
    1152                 :            : 
    1153                 :            :                 /* Let the USB core know _both_ roothubs lost power. */
    1154                 :          0 :                 usb_root_hub_lost_power(xhci->main_hcd->self.root_hub);
    1155                 :          0 :                 usb_root_hub_lost_power(xhci->shared_hcd->self.root_hub);
    1156                 :            : 
    1157                 :          0 :                 xhci_dbg(xhci, "Stop HCD\n");
    1158                 :          0 :                 xhci_halt(xhci);
    1159                 :          0 :                 xhci_zero_64b_regs(xhci);
    1160                 :          0 :                 xhci_reset(xhci);
    1161                 :          0 :                 spin_unlock_irq(&xhci->lock);
    1162                 :          0 :                 xhci_cleanup_msix(xhci);
    1163                 :            : 
    1164                 :          0 :                 xhci_dbg(xhci, "// Disabling event ring interrupts\n");
    1165                 :          0 :                 temp = readl(&xhci->op_regs->status);
    1166                 :          0 :                 writel((temp & ~0x1fff) | STS_EINT, &xhci->op_regs->status);
    1167                 :          0 :                 temp = readl(&xhci->ir_set->irq_pending);
    1168                 :          0 :                 writel(ER_IRQ_DISABLE(temp), &xhci->ir_set->irq_pending);
    1169                 :            : 
    1170                 :          0 :                 xhci_dbg(xhci, "cleaning up memory\n");
    1171                 :          0 :                 xhci_mem_cleanup(xhci);
    1172                 :          0 :                 xhci_debugfs_exit(xhci);
    1173                 :          0 :                 xhci_dbg(xhci, "xhci_stop completed - status = %x\n",
    1174                 :            :                             readl(&xhci->op_regs->status));
    1175                 :            : 
    1176                 :            :                 /* USB core calls the PCI reinit and start functions twice:
    1177                 :            :                  * first with the primary HCD, and then with the secondary HCD.
    1178                 :            :                  * If we don't do the same, the host will never be started.
    1179                 :            :                  */
    1180         [ #  # ]:          0 :                 if (!usb_hcd_is_primary_hcd(hcd))
    1181                 :            :                         secondary_hcd = hcd;
    1182                 :            :                 else
    1183                 :          0 :                         secondary_hcd = xhci->shared_hcd;
    1184                 :            : 
    1185                 :          0 :                 xhci_dbg(xhci, "Initialize the xhci_hcd\n");
    1186                 :          0 :                 retval = xhci_init(hcd->primary_hcd);
    1187         [ #  # ]:          0 :                 if (retval)
    1188                 :            :                         return retval;
    1189                 :          0 :                 comp_timer_running = true;
    1190                 :            : 
    1191                 :          0 :                 xhci_dbg(xhci, "Start the primary HCD\n");
    1192                 :          0 :                 retval = xhci_run(hcd->primary_hcd);
    1193         [ #  # ]:          0 :                 if (!retval) {
    1194                 :          0 :                         xhci_dbg(xhci, "Start the secondary HCD\n");
    1195                 :          0 :                         retval = xhci_run(secondary_hcd);
    1196                 :            :                 }
    1197                 :          0 :                 hcd->state = HC_STATE_SUSPENDED;
    1198                 :          0 :                 xhci->shared_hcd->state = HC_STATE_SUSPENDED;
    1199                 :          0 :                 goto done;
    1200                 :            :         }
    1201                 :            : 
    1202                 :            :         /* step 4: set Run/Stop bit */
    1203                 :          0 :         command = readl(&xhci->op_regs->command);
    1204                 :          0 :         command |= CMD_RUN;
    1205                 :          0 :         writel(command, &xhci->op_regs->command);
    1206                 :          0 :         xhci_handshake(&xhci->op_regs->status, STS_HALT,
    1207                 :            :                   0, 250 * 1000);
    1208                 :            : 
    1209                 :            :         /* step 5: walk topology and initialize portsc,
    1210                 :            :          * portpmsc and portli
    1211                 :            :          */
    1212                 :            :         /* this is done in bus_resume */
    1213                 :            : 
    1214                 :            :         /* step 6: restart each of the previously
    1215                 :            :          * Running endpoints by ringing their doorbells
    1216                 :            :          */
    1217                 :            : 
    1218                 :          0 :         spin_unlock_irq(&xhci->lock);
    1219                 :            : 
    1220                 :          0 :         xhci_dbc_resume(xhci);
    1221                 :            : 
    1222                 :          0 :  done:
    1223         [ #  # ]:          0 :         if (retval == 0) {
    1224                 :            :                 /* Resume root hubs only when have pending events. */
    1225         [ #  # ]:          0 :                 if (xhci_pending_portevent(xhci)) {
    1226                 :          0 :                         usb_hcd_resume_root_hub(xhci->shared_hcd);
    1227                 :          0 :                         usb_hcd_resume_root_hub(hcd);
    1228                 :            :                 }
    1229                 :            :         }
    1230                 :            : 
    1231                 :            :         /*
    1232                 :            :          * If system is subject to the Quirk, Compliance Mode Timer needs to
    1233                 :            :          * be re-initialized Always after a system resume. Ports are subject
    1234                 :            :          * to suffer the Compliance Mode issue again. It doesn't matter if
    1235                 :            :          * ports have entered previously to U0 before system's suspension.
    1236                 :            :          */
    1237   [ #  #  #  # ]:          0 :         if ((xhci->quirks & XHCI_COMP_MODE_QUIRK) && !comp_timer_running)
    1238                 :          0 :                 compliance_mode_recovery_timer_init(xhci);
    1239                 :            : 
    1240         [ #  # ]:          0 :         if (xhci->quirks & XHCI_ASMEDIA_MODIFY_FLOWCONTROL)
    1241                 :          0 :                 usb_asmedia_modifyflowcontrol(to_pci_dev(hcd->self.controller));
    1242                 :            : 
    1243                 :            :         /* Re-enable port polling. */
    1244                 :          0 :         xhci_dbg(xhci, "%s: starting port polling.\n", __func__);
    1245                 :          0 :         set_bit(HCD_FLAG_POLL_RH, &xhci->shared_hcd->flags);
    1246                 :          0 :         usb_hcd_poll_rh_status(xhci->shared_hcd);
    1247                 :          0 :         set_bit(HCD_FLAG_POLL_RH, &hcd->flags);
    1248                 :          0 :         usb_hcd_poll_rh_status(hcd);
    1249                 :            : 
    1250                 :          0 :         return retval;
    1251                 :            : }
    1252                 :            : EXPORT_SYMBOL_GPL(xhci_resume);
    1253                 :            : #endif  /* CONFIG_PM */
    1254                 :            : 
    1255                 :            : /*-------------------------------------------------------------------------*/
    1256                 :            : 
    1257                 :            : /*
    1258                 :            :  * Bypass the DMA mapping if URB is suitable for Immediate Transfer (IDT),
    1259                 :            :  * we'll copy the actual data into the TRB address register. This is limited to
    1260                 :            :  * transfers up to 8 bytes on output endpoints of any kind with wMaxPacketSize
    1261                 :            :  * >= 8 bytes. If suitable for IDT only one Transfer TRB per TD is allowed.
    1262                 :            :  */
    1263                 :          0 : static int xhci_map_urb_for_dma(struct usb_hcd *hcd, struct urb *urb,
    1264                 :            :                                 gfp_t mem_flags)
    1265                 :            : {
    1266         [ #  # ]:          0 :         if (xhci_urb_suitable_for_idt(urb))
    1267                 :            :                 return 0;
    1268                 :            : 
    1269                 :          0 :         return usb_hcd_map_urb_for_dma(hcd, urb, mem_flags);
    1270                 :            : }
    1271                 :            : 
    1272                 :            : /**
    1273                 :            :  * xhci_get_endpoint_index - Used for passing endpoint bitmasks between the core and
    1274                 :            :  * HCDs.  Find the index for an endpoint given its descriptor.  Use the return
    1275                 :            :  * value to right shift 1 for the bitmask.
    1276                 :            :  *
    1277                 :            :  * Index  = (epnum * 2) + direction - 1,
    1278                 :            :  * where direction = 0 for OUT, 1 for IN.
    1279                 :            :  * For control endpoints, the IN index is used (OUT index is unused), so
    1280                 :            :  * index = (epnum * 2) + direction - 1 = (epnum * 2) + 1 - 1 = (epnum * 2)
    1281                 :            :  */
    1282                 :          0 : unsigned int xhci_get_endpoint_index(struct usb_endpoint_descriptor *desc)
    1283                 :            : {
    1284                 :          0 :         unsigned int index;
    1285   [ #  #  #  # ]:          0 :         if (usb_endpoint_xfer_control(desc))
    1286                 :          0 :                 index = (unsigned int) (usb_endpoint_num(desc)*2);
    1287                 :            :         else
    1288   [ #  #  #  #  :          0 :                 index = (unsigned int) (usb_endpoint_num(desc)*2) +
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1289   [ #  #  #  #  :          0 :                         (usb_endpoint_dir_in(desc) ? 1 : 0) - 1;
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
          #  #  #  #  #  
                   #  # ]
    1290                 :          0 :         return index;
    1291                 :            : }
    1292                 :            : 
    1293                 :            : /* The reverse operation to xhci_get_endpoint_index. Calculate the USB endpoint
    1294                 :            :  * address from the XHCI endpoint index.
    1295                 :            :  */
    1296                 :          0 : unsigned int xhci_get_endpoint_address(unsigned int ep_index)
    1297                 :            : {
    1298                 :          0 :         unsigned int number = DIV_ROUND_UP(ep_index, 2);
    1299         [ #  # ]:          0 :         unsigned int direction = ep_index % 2 ? USB_DIR_OUT : USB_DIR_IN;
    1300                 :          0 :         return direction | number;
    1301                 :            : }
    1302                 :            : 
    1303                 :            : /* Find the flag for this endpoint (for use in the control context).  Use the
    1304                 :            :  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
    1305                 :            :  * bit 1, etc.
    1306                 :            :  */
    1307                 :          0 : static unsigned int xhci_get_endpoint_flag(struct usb_endpoint_descriptor *desc)
    1308                 :            : {
    1309                 :          0 :         return 1 << (xhci_get_endpoint_index(desc) + 1);
    1310                 :            : }
    1311                 :            : 
    1312                 :            : /* Find the flag for this endpoint (for use in the control context).  Use the
    1313                 :            :  * endpoint index to create a bitmask.  The slot context is bit 0, endpoint 0 is
    1314                 :            :  * bit 1, etc.
    1315                 :            :  */
    1316                 :          0 : static unsigned int xhci_get_endpoint_flag_from_index(unsigned int ep_index)
    1317                 :            : {
    1318                 :          0 :         return 1 << (ep_index + 1);
    1319                 :            : }
    1320                 :            : 
    1321                 :            : /* Compute the last valid endpoint context index.  Basically, this is the
    1322                 :            :  * endpoint index plus one.  For slot contexts with more than valid endpoint,
    1323                 :            :  * we find the most significant bit set in the added contexts flags.
    1324                 :            :  * e.g. ep 1 IN (with epnum 0x81) => added_ctxs = 0b1000
    1325                 :            :  * fls(0b1000) = 4, but the endpoint context index is 3, so subtract one.
    1326                 :            :  */
    1327                 :          0 : unsigned int xhci_last_valid_endpoint(u32 added_ctxs)
    1328                 :            : {
    1329                 :          0 :         return fls(added_ctxs) - 1;
    1330                 :            : }
    1331                 :            : 
    1332                 :            : /* Returns 1 if the arguments are OK;
    1333                 :            :  * returns 0 this is a root hub; returns -EINVAL for NULL pointers.
    1334                 :            :  */
    1335                 :          0 : static int xhci_check_args(struct usb_hcd *hcd, struct usb_device *udev,
    1336                 :            :                 struct usb_host_endpoint *ep, int check_ep, bool check_virt_dev,
    1337                 :            :                 const char *func) {
    1338                 :          0 :         struct xhci_hcd *xhci;
    1339                 :          0 :         struct xhci_virt_device *virt_dev;
    1340                 :            : 
    1341   [ #  #  #  #  :          0 :         if (!hcd || (check_ep && !ep) || !udev) {
                   #  # ]
    1342                 :            :                 pr_debug("xHCI %s called with invalid args\n", func);
    1343                 :            :                 return -EINVAL;
    1344                 :            :         }
    1345         [ #  # ]:          0 :         if (!udev->parent) {
    1346                 :            :                 pr_debug("xHCI %s called for root hub\n", func);
    1347                 :            :                 return 0;
    1348                 :            :         }
    1349                 :            : 
    1350                 :          0 :         xhci = hcd_to_xhci(hcd);
    1351         [ #  # ]:          0 :         if (check_virt_dev) {
    1352   [ #  #  #  # ]:          0 :                 if (!udev->slot_id || !xhci->devs[udev->slot_id]) {
    1353                 :            :                         xhci_dbg(xhci, "xHCI %s called with unaddressed device\n",
    1354                 :            :                                         func);
    1355                 :            :                         return -EINVAL;
    1356                 :            :                 }
    1357                 :            : 
    1358                 :          0 :                 virt_dev = xhci->devs[udev->slot_id];
    1359         [ #  # ]:          0 :                 if (virt_dev->udev != udev) {
    1360                 :            :                         xhci_dbg(xhci, "xHCI %s called with udev and "
    1361                 :            :                                           "virt_dev does not match\n", func);
    1362                 :            :                         return -EINVAL;
    1363                 :            :                 }
    1364                 :            :         }
    1365                 :            : 
    1366         [ #  # ]:          0 :         if (xhci->xhc_state & XHCI_STATE_HALTED)
    1367                 :          0 :                 return -ENODEV;
    1368                 :            : 
    1369                 :            :         return 1;
    1370                 :            : }
    1371                 :            : 
    1372                 :            : static int xhci_configure_endpoint(struct xhci_hcd *xhci,
    1373                 :            :                 struct usb_device *udev, struct xhci_command *command,
    1374                 :            :                 bool ctx_change, bool must_succeed);
    1375                 :            : 
    1376                 :            : /*
    1377                 :            :  * Full speed devices may have a max packet size greater than 8 bytes, but the
    1378                 :            :  * USB core doesn't know that until it reads the first 8 bytes of the
    1379                 :            :  * descriptor.  If the usb_device's max packet size changes after that point,
    1380                 :            :  * we need to issue an evaluate context command and wait on it.
    1381                 :            :  */
    1382                 :            : static int xhci_check_maxpacket(struct xhci_hcd *xhci, unsigned int slot_id,
    1383                 :            :                 unsigned int ep_index, struct urb *urb)
    1384                 :            : {
    1385                 :            :         struct xhci_container_ctx *out_ctx;
    1386                 :            :         struct xhci_input_control_ctx *ctrl_ctx;
    1387                 :            :         struct xhci_ep_ctx *ep_ctx;
    1388                 :            :         struct xhci_command *command;
    1389                 :            :         int max_packet_size;
    1390                 :            :         int hw_max_packet_size;
    1391                 :            :         int ret = 0;
    1392                 :            : 
    1393                 :            :         out_ctx = xhci->devs[slot_id]->out_ctx;
    1394                 :            :         ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
    1395                 :            :         hw_max_packet_size = MAX_PACKET_DECODED(le32_to_cpu(ep_ctx->ep_info2));
    1396                 :            :         max_packet_size = usb_endpoint_maxp(&urb->dev->ep0.desc);
    1397                 :            :         if (hw_max_packet_size != max_packet_size) {
    1398                 :            :                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
    1399                 :            :                                 "Max Packet Size for ep 0 changed.");
    1400                 :            :                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
    1401                 :            :                                 "Max packet size in usb_device = %d",
    1402                 :            :                                 max_packet_size);
    1403                 :            :                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
    1404                 :            :                                 "Max packet size in xHCI HW = %d",
    1405                 :            :                                 hw_max_packet_size);
    1406                 :            :                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
    1407                 :            :                                 "Issuing evaluate context command.");
    1408                 :            : 
    1409                 :            :                 /* Set up the input context flags for the command */
    1410                 :            :                 /* FIXME: This won't work if a non-default control endpoint
    1411                 :            :                  * changes max packet sizes.
    1412                 :            :                  */
    1413                 :            : 
    1414                 :            :                 command = xhci_alloc_command(xhci, true, GFP_KERNEL);
    1415                 :            :                 if (!command)
    1416                 :            :                         return -ENOMEM;
    1417                 :            : 
    1418                 :            :                 command->in_ctx = xhci->devs[slot_id]->in_ctx;
    1419                 :            :                 ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
    1420                 :            :                 if (!ctrl_ctx) {
    1421                 :            :                         xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    1422                 :            :                                         __func__);
    1423                 :            :                         ret = -ENOMEM;
    1424                 :            :                         goto command_cleanup;
    1425                 :            :                 }
    1426                 :            :                 /* Set up the modified control endpoint 0 */
    1427                 :            :                 xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
    1428                 :            :                                 xhci->devs[slot_id]->out_ctx, ep_index);
    1429                 :            : 
    1430                 :            :                 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
    1431                 :            :                 ep_ctx->ep_info2 &= cpu_to_le32(~MAX_PACKET_MASK);
    1432                 :            :                 ep_ctx->ep_info2 |= cpu_to_le32(MAX_PACKET(max_packet_size));
    1433                 :            : 
    1434                 :            :                 ctrl_ctx->add_flags = cpu_to_le32(EP0_FLAG);
    1435                 :            :                 ctrl_ctx->drop_flags = 0;
    1436                 :            : 
    1437                 :            :                 ret = xhci_configure_endpoint(xhci, urb->dev, command,
    1438                 :            :                                 true, false);
    1439                 :            : 
    1440                 :            :                 /* Clean up the input context for later use by bandwidth
    1441                 :            :                  * functions.
    1442                 :            :                  */
    1443                 :            :                 ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG);
    1444                 :            : command_cleanup:
    1445                 :            :                 kfree(command->completion);
    1446                 :            :                 kfree(command);
    1447                 :            :         }
    1448                 :            :         return ret;
    1449                 :            : }
    1450                 :            : 
    1451                 :            : /*
    1452                 :            :  * non-error returns are a promise to giveback() the urb later
    1453                 :            :  * we drop ownership so next owner (or urb unlink) can get it
    1454                 :            :  */
    1455                 :          0 : static int xhci_urb_enqueue(struct usb_hcd *hcd, struct urb *urb, gfp_t mem_flags)
    1456                 :            : {
    1457                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    1458                 :          0 :         unsigned long flags;
    1459                 :          0 :         int ret = 0;
    1460                 :          0 :         unsigned int slot_id, ep_index;
    1461                 :          0 :         unsigned int *ep_state;
    1462                 :          0 :         struct urb_priv *urb_priv;
    1463                 :          0 :         int num_tds;
    1464                 :            : 
    1465   [ #  #  #  # ]:          0 :         if (!urb || xhci_check_args(hcd, urb->dev, urb->ep,
    1466                 :            :                                         true, true, __func__) <= 0)
    1467                 :          0 :                 return -EINVAL;
    1468                 :            : 
    1469                 :          0 :         slot_id = urb->dev->slot_id;
    1470         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&urb->ep->desc);
    1471                 :          0 :         ep_state = &xhci->devs[slot_id]->eps[ep_index].ep_state;
    1472                 :            : 
    1473         [ #  # ]:          0 :         if (!HCD_HW_ACCESSIBLE(hcd)) {
    1474                 :          0 :                 if (!in_interrupt())
    1475                 :            :                         xhci_dbg(xhci, "urb submitted during PCI suspend\n");
    1476                 :          0 :                 return -ESHUTDOWN;
    1477                 :            :         }
    1478         [ #  # ]:          0 :         if (xhci->devs[slot_id]->flags & VDEV_PORT_ERROR) {
    1479                 :            :                 xhci_dbg(xhci, "Can't queue urb, port error, link inactive\n");
    1480                 :            :                 return -ENODEV;
    1481                 :            :         }
    1482                 :            : 
    1483         [ #  # ]:          0 :         if (usb_endpoint_xfer_isoc(&urb->ep->desc))
    1484                 :          0 :                 num_tds = urb->number_of_packets;
    1485         [ #  # ]:          0 :         else if (usb_endpoint_is_bulk_out(&urb->ep->desc) &&
    1486         [ #  # ]:          0 :             urb->transfer_buffer_length > 0 &&
    1487         [ #  # ]:          0 :             urb->transfer_flags & URB_ZERO_PACKET &&
    1488         [ #  # ]:          0 :             !(urb->transfer_buffer_length % usb_endpoint_maxp(&urb->ep->desc)))
    1489                 :            :                 num_tds = 2;
    1490                 :            :         else
    1491                 :            :                 num_tds = 1;
    1492                 :            : 
    1493         [ #  # ]:          0 :         urb_priv = kzalloc(struct_size(urb_priv, td, num_tds), mem_flags);
    1494         [ #  # ]:          0 :         if (!urb_priv)
    1495                 :            :                 return -ENOMEM;
    1496                 :            : 
    1497                 :          0 :         urb_priv->num_tds = num_tds;
    1498                 :          0 :         urb_priv->num_tds_done = 0;
    1499                 :          0 :         urb->hcpriv = urb_priv;
    1500                 :            : 
    1501                 :          0 :         trace_xhci_urb_enqueue(urb);
    1502                 :            : 
    1503         [ #  # ]:          0 :         if (usb_endpoint_xfer_control(&urb->ep->desc)) {
    1504                 :            :                 /* Check to see if the max packet size for the default control
    1505                 :            :                  * endpoint changed during FS device enumeration
    1506                 :            :                  */
    1507         [ #  # ]:          0 :                 if (urb->dev->speed == USB_SPEED_FULL) {
    1508                 :          0 :                         ret = xhci_check_maxpacket(xhci, slot_id,
    1509                 :            :                                         ep_index, urb);
    1510         [ #  # ]:          0 :                         if (ret < 0) {
    1511                 :          0 :                                 xhci_urb_free_priv(urb_priv);
    1512                 :          0 :                                 urb->hcpriv = NULL;
    1513                 :          0 :                                 return ret;
    1514                 :            :                         }
    1515                 :            :                 }
    1516                 :            :         }
    1517                 :            : 
    1518                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    1519                 :            : 
    1520         [ #  # ]:          0 :         if (xhci->xhc_state & XHCI_STATE_DYING) {
    1521                 :          0 :                 xhci_dbg(xhci, "Ep 0x%x: URB %p submitted for non-responsive xHCI host.\n",
    1522                 :            :                          urb->ep->desc.bEndpointAddress, urb);
    1523                 :          0 :                 ret = -ESHUTDOWN;
    1524                 :          0 :                 goto free_priv;
    1525                 :            :         }
    1526         [ #  # ]:          0 :         if (*ep_state & (EP_GETTING_STREAMS | EP_GETTING_NO_STREAMS)) {
    1527                 :          0 :                 xhci_warn(xhci, "WARN: Can't enqueue URB, ep in streams transition state %x\n",
    1528                 :            :                           *ep_state);
    1529                 :          0 :                 ret = -EINVAL;
    1530                 :          0 :                 goto free_priv;
    1531                 :            :         }
    1532         [ #  # ]:          0 :         if (*ep_state & EP_SOFT_CLEAR_TOGGLE) {
    1533                 :          0 :                 xhci_warn(xhci, "Can't enqueue URB while manually clearing toggle\n");
    1534                 :          0 :                 ret = -EINVAL;
    1535                 :          0 :                 goto free_priv;
    1536                 :            :         }
    1537                 :            : 
    1538   [ #  #  #  # ]:          0 :         switch (usb_endpoint_type(&urb->ep->desc)) {
    1539                 :            : 
    1540                 :          0 :         case USB_ENDPOINT_XFER_CONTROL:
    1541                 :          0 :                 ret = xhci_queue_ctrl_tx(xhci, GFP_ATOMIC, urb,
    1542                 :            :                                          slot_id, ep_index);
    1543                 :          0 :                 break;
    1544                 :          0 :         case USB_ENDPOINT_XFER_BULK:
    1545                 :          0 :                 ret = xhci_queue_bulk_tx(xhci, GFP_ATOMIC, urb,
    1546                 :            :                                          slot_id, ep_index);
    1547                 :          0 :                 break;
    1548                 :          0 :         case USB_ENDPOINT_XFER_INT:
    1549                 :          0 :                 ret = xhci_queue_intr_tx(xhci, GFP_ATOMIC, urb,
    1550                 :            :                                 slot_id, ep_index);
    1551                 :          0 :                 break;
    1552                 :          0 :         case USB_ENDPOINT_XFER_ISOC:
    1553                 :          0 :                 ret = xhci_queue_isoc_tx_prepare(xhci, GFP_ATOMIC, urb,
    1554                 :            :                                 slot_id, ep_index);
    1555                 :            :         }
    1556                 :            : 
    1557         [ #  # ]:          0 :         if (ret) {
    1558                 :          0 : free_priv:
    1559                 :          0 :                 xhci_urb_free_priv(urb_priv);
    1560                 :          0 :                 urb->hcpriv = NULL;
    1561                 :            :         }
    1562                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    1563                 :          0 :         return ret;
    1564                 :            : }
    1565                 :            : 
    1566                 :            : /*
    1567                 :            :  * Remove the URB's TD from the endpoint ring.  This may cause the HC to stop
    1568                 :            :  * USB transfers, potentially stopping in the middle of a TRB buffer.  The HC
    1569                 :            :  * should pick up where it left off in the TD, unless a Set Transfer Ring
    1570                 :            :  * Dequeue Pointer is issued.
    1571                 :            :  *
    1572                 :            :  * The TRBs that make up the buffers for the canceled URB will be "removed" from
    1573                 :            :  * the ring.  Since the ring is a contiguous structure, they can't be physically
    1574                 :            :  * removed.  Instead, there are two options:
    1575                 :            :  *
    1576                 :            :  *  1) If the HC is in the middle of processing the URB to be canceled, we
    1577                 :            :  *     simply move the ring's dequeue pointer past those TRBs using the Set
    1578                 :            :  *     Transfer Ring Dequeue Pointer command.  This will be the common case,
    1579                 :            :  *     when drivers timeout on the last submitted URB and attempt to cancel.
    1580                 :            :  *
    1581                 :            :  *  2) If the HC is in the middle of a different TD, we turn the TRBs into a
    1582                 :            :  *     series of 1-TRB transfer no-op TDs.  (No-ops shouldn't be chained.)  The
    1583                 :            :  *     HC will need to invalidate the any TRBs it has cached after the stop
    1584                 :            :  *     endpoint command, as noted in the xHCI 0.95 errata.
    1585                 :            :  *
    1586                 :            :  *  3) The TD may have completed by the time the Stop Endpoint Command
    1587                 :            :  *     completes, so software needs to handle that case too.
    1588                 :            :  *
    1589                 :            :  * This function should protect against the TD enqueueing code ringing the
    1590                 :            :  * doorbell while this code is waiting for a Stop Endpoint command to complete.
    1591                 :            :  * It also needs to account for multiple cancellations on happening at the same
    1592                 :            :  * time for the same endpoint.
    1593                 :            :  *
    1594                 :            :  * Note that this function can be called in any context, or so says
    1595                 :            :  * usb_hcd_unlink_urb()
    1596                 :            :  */
    1597                 :          0 : static int xhci_urb_dequeue(struct usb_hcd *hcd, struct urb *urb, int status)
    1598                 :            : {
    1599                 :          0 :         unsigned long flags;
    1600                 :          0 :         int ret, i;
    1601                 :          0 :         u32 temp;
    1602                 :          0 :         struct xhci_hcd *xhci;
    1603                 :          0 :         struct urb_priv *urb_priv;
    1604                 :          0 :         struct xhci_td *td;
    1605                 :          0 :         unsigned int ep_index;
    1606                 :          0 :         struct xhci_ring *ep_ring;
    1607                 :          0 :         struct xhci_virt_ep *ep;
    1608                 :          0 :         struct xhci_command *command;
    1609                 :          0 :         struct xhci_virt_device *vdev;
    1610                 :            : 
    1611                 :          0 :         xhci = hcd_to_xhci(hcd);
    1612                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    1613                 :            : 
    1614                 :          0 :         trace_xhci_urb_dequeue(urb);
    1615                 :            : 
    1616                 :            :         /* Make sure the URB hasn't completed or been unlinked already */
    1617                 :          0 :         ret = usb_hcd_check_unlink_urb(hcd, urb, status);
    1618         [ #  # ]:          0 :         if (ret)
    1619                 :          0 :                 goto done;
    1620                 :            : 
    1621                 :            :         /* give back URB now if we can't queue it for cancel */
    1622                 :          0 :         vdev = xhci->devs[urb->dev->slot_id];
    1623                 :          0 :         urb_priv = urb->hcpriv;
    1624         [ #  # ]:          0 :         if (!vdev || !urb_priv)
    1625                 :          0 :                 goto err_giveback;
    1626                 :            : 
    1627         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&urb->ep->desc);
    1628                 :          0 :         ep = &vdev->eps[ep_index];
    1629                 :          0 :         ep_ring = xhci_urb_to_transfer_ring(xhci, urb);
    1630         [ #  # ]:          0 :         if (!ep || !ep_ring)
    1631                 :          0 :                 goto err_giveback;
    1632                 :            : 
    1633                 :            :         /* If xHC is dead take it down and return ALL URBs in xhci_hc_died() */
    1634                 :          0 :         temp = readl(&xhci->op_regs->status);
    1635   [ #  #  #  # ]:          0 :         if (temp == ~(u32)0 || xhci->xhc_state & XHCI_STATE_DYING) {
    1636                 :          0 :                 xhci_hc_died(xhci);
    1637                 :          0 :                 goto done;
    1638                 :            :         }
    1639                 :            : 
    1640                 :            :         /*
    1641                 :            :          * check ring is not re-allocated since URB was enqueued. If it is, then
    1642                 :            :          * make sure none of the ring related pointers in this URB private data
    1643                 :            :          * are touched, such as td_list, otherwise we overwrite freed data
    1644                 :            :          */
    1645   [ #  #  #  # ]:          0 :         if (!td_on_ring(&urb_priv->td[0], ep_ring)) {
    1646                 :          0 :                 xhci_err(xhci, "Canceled URB td not found on endpoint ring");
    1647         [ #  # ]:          0 :                 for (i = urb_priv->num_tds_done; i < urb_priv->num_tds; i++) {
    1648                 :          0 :                         td = &urb_priv->td[i];
    1649         [ #  # ]:          0 :                         if (!list_empty(&td->cancelled_td_list))
    1650                 :          0 :                                 list_del_init(&td->cancelled_td_list);
    1651                 :            :                 }
    1652                 :          0 :                 goto err_giveback;
    1653                 :            :         }
    1654                 :            : 
    1655         [ #  # ]:          0 :         if (xhci->xhc_state & XHCI_STATE_HALTED) {
    1656                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
    1657                 :            :                                 "HC halted, freeing TD manually.");
    1658                 :          0 :                 for (i = urb_priv->num_tds_done;
    1659         [ #  # ]:          0 :                      i < urb_priv->num_tds;
    1660                 :          0 :                      i++) {
    1661                 :          0 :                         td = &urb_priv->td[i];
    1662         [ #  # ]:          0 :                         if (!list_empty(&td->td_list))
    1663                 :          0 :                                 list_del_init(&td->td_list);
    1664         [ #  # ]:          0 :                         if (!list_empty(&td->cancelled_td_list))
    1665                 :          0 :                                 list_del_init(&td->cancelled_td_list);
    1666                 :            :                 }
    1667                 :          0 :                 goto err_giveback;
    1668                 :            :         }
    1669                 :            : 
    1670                 :          0 :         i = urb_priv->num_tds_done;
    1671         [ #  # ]:          0 :         if (i < urb_priv->num_tds)
    1672                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_cancel_urb,
    1673                 :            :                                 "Cancel URB %p, dev %s, ep 0x%x, "
    1674                 :            :                                 "starting at offset 0x%llx",
    1675                 :          0 :                                 urb, urb->dev->devpath,
    1676                 :          0 :                                 urb->ep->desc.bEndpointAddress,
    1677                 :          0 :                                 (unsigned long long) xhci_trb_virt_to_dma(
    1678                 :            :                                         urb_priv->td[i].start_seg,
    1679                 :            :                                         urb_priv->td[i].first_trb));
    1680                 :            : 
    1681         [ #  # ]:          0 :         for (; i < urb_priv->num_tds; i++) {
    1682                 :          0 :                 td = &urb_priv->td[i];
    1683                 :          0 :                 list_add_tail(&td->cancelled_td_list, &ep->cancelled_td_list);
    1684                 :            :         }
    1685                 :            : 
    1686                 :            :         /* Queue a stop endpoint command, but only if this is
    1687                 :            :          * the first cancellation to be handled.
    1688                 :            :          */
    1689         [ #  # ]:          0 :         if (!(ep->ep_state & EP_STOP_CMD_PENDING)) {
    1690                 :          0 :                 command = xhci_alloc_command(xhci, false, GFP_ATOMIC);
    1691         [ #  # ]:          0 :                 if (!command) {
    1692                 :          0 :                         ret = -ENOMEM;
    1693                 :          0 :                         goto done;
    1694                 :            :                 }
    1695                 :          0 :                 ep->ep_state |= EP_STOP_CMD_PENDING;
    1696                 :          0 :                 ep->stop_cmd_timer.expires = jiffies +
    1697                 :            :                         XHCI_STOP_EP_CMD_TIMEOUT * HZ;
    1698                 :          0 :                 add_timer(&ep->stop_cmd_timer);
    1699                 :          0 :                 xhci_queue_stop_endpoint(xhci, command, urb->dev->slot_id,
    1700                 :            :                                          ep_index, 0);
    1701                 :          0 :                 xhci_ring_cmd_db(xhci);
    1702                 :            :         }
    1703                 :          0 : done:
    1704                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    1705                 :          0 :         return ret;
    1706                 :            : 
    1707                 :          0 : err_giveback:
    1708         [ #  # ]:          0 :         if (urb_priv)
    1709                 :          0 :                 xhci_urb_free_priv(urb_priv);
    1710                 :          0 :         usb_hcd_unlink_urb_from_ep(hcd, urb);
    1711                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    1712                 :          0 :         usb_hcd_giveback_urb(hcd, urb, -ESHUTDOWN);
    1713                 :          0 :         return ret;
    1714                 :            : }
    1715                 :            : 
    1716                 :            : /* Drop an endpoint from a new bandwidth configuration for this device.
    1717                 :            :  * Only one call to this function is allowed per endpoint before
    1718                 :            :  * check_bandwidth() or reset_bandwidth() must be called.
    1719                 :            :  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
    1720                 :            :  * add the endpoint to the schedule with possibly new parameters denoted by a
    1721                 :            :  * different endpoint descriptor in usb_host_endpoint.
    1722                 :            :  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
    1723                 :            :  * not allowed.
    1724                 :            :  *
    1725                 :            :  * The USB core will not allow URBs to be queued to an endpoint that is being
    1726                 :            :  * disabled, so there's no need for mutual exclusion to protect
    1727                 :            :  * the xhci->devs[slot_id] structure.
    1728                 :            :  */
    1729                 :          0 : static int xhci_drop_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
    1730                 :            :                 struct usb_host_endpoint *ep)
    1731                 :            : {
    1732                 :          0 :         struct xhci_hcd *xhci;
    1733                 :          0 :         struct xhci_container_ctx *in_ctx, *out_ctx;
    1734                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    1735                 :          0 :         unsigned int ep_index;
    1736                 :          0 :         struct xhci_ep_ctx *ep_ctx;
    1737                 :          0 :         u32 drop_flag;
    1738                 :          0 :         u32 new_add_flags, new_drop_flags;
    1739                 :          0 :         int ret;
    1740                 :            : 
    1741                 :          0 :         ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
    1742         [ #  # ]:          0 :         if (ret <= 0)
    1743                 :            :                 return ret;
    1744                 :          0 :         xhci = hcd_to_xhci(hcd);
    1745         [ #  # ]:          0 :         if (xhci->xhc_state & XHCI_STATE_DYING)
    1746                 :            :                 return -ENODEV;
    1747                 :            : 
    1748                 :          0 :         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
    1749         [ #  # ]:          0 :         drop_flag = xhci_get_endpoint_flag(&ep->desc);
    1750         [ #  # ]:          0 :         if (drop_flag == SLOT_FLAG || drop_flag == EP0_FLAG) {
    1751                 :            :                 xhci_dbg(xhci, "xHCI %s - can't drop slot or ep 0 %#x\n",
    1752                 :            :                                 __func__, drop_flag);
    1753                 :            :                 return 0;
    1754                 :            :         }
    1755                 :            : 
    1756                 :          0 :         in_ctx = xhci->devs[udev->slot_id]->in_ctx;
    1757                 :          0 :         out_ctx = xhci->devs[udev->slot_id]->out_ctx;
    1758                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
    1759         [ #  # ]:          0 :         if (!ctrl_ctx) {
    1760                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    1761                 :            :                                 __func__);
    1762                 :          0 :                 return 0;
    1763                 :            :         }
    1764                 :            : 
    1765         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&ep->desc);
    1766                 :          0 :         ep_ctx = xhci_get_ep_ctx(xhci, out_ctx, ep_index);
    1767                 :            :         /* If the HC already knows the endpoint is disabled,
    1768                 :            :          * or the HCD has noted it is disabled, ignore this request
    1769                 :            :          */
    1770         [ #  # ]:          0 :         if ((GET_EP_CTX_STATE(ep_ctx) == EP_STATE_DISABLED) ||
    1771   [ #  #  #  # ]:          0 :             le32_to_cpu(ctrl_ctx->drop_flags) &
    1772                 :            :             xhci_get_endpoint_flag(&ep->desc)) {
    1773                 :            :                 /* Do not warn when called after a usb_device_reset */
    1774         [ #  # ]:          0 :                 if (xhci->devs[udev->slot_id]->eps[ep_index].ring != NULL)
    1775                 :          0 :                         xhci_warn(xhci, "xHCI %s called with disabled ep %p\n",
    1776                 :            :                                   __func__, ep);
    1777                 :          0 :                 return 0;
    1778                 :            :         }
    1779                 :            : 
    1780                 :          0 :         ctrl_ctx->drop_flags |= cpu_to_le32(drop_flag);
    1781                 :          0 :         new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
    1782                 :            : 
    1783                 :          0 :         ctrl_ctx->add_flags &= cpu_to_le32(~drop_flag);
    1784                 :          0 :         new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
    1785                 :            : 
    1786                 :          0 :         xhci_debugfs_remove_endpoint(xhci, xhci->devs[udev->slot_id], ep_index);
    1787                 :            : 
    1788                 :          0 :         xhci_endpoint_zero(xhci, xhci->devs[udev->slot_id], ep);
    1789                 :            : 
    1790                 :          0 :         if (xhci->quirks & XHCI_MTK_HOST)
    1791                 :            :                 xhci_mtk_drop_ep_quirk(hcd, udev, ep);
    1792                 :            : 
    1793                 :          0 :         xhci_dbg(xhci, "drop ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
    1794                 :            :                         (unsigned int) ep->desc.bEndpointAddress,
    1795                 :            :                         udev->slot_id,
    1796                 :            :                         (unsigned int) new_drop_flags,
    1797                 :            :                         (unsigned int) new_add_flags);
    1798                 :          0 :         return 0;
    1799                 :            : }
    1800                 :            : 
    1801                 :            : /* Add an endpoint to a new possible bandwidth configuration for this device.
    1802                 :            :  * Only one call to this function is allowed per endpoint before
    1803                 :            :  * check_bandwidth() or reset_bandwidth() must be called.
    1804                 :            :  * A call to xhci_drop_endpoint() followed by a call to xhci_add_endpoint() will
    1805                 :            :  * add the endpoint to the schedule with possibly new parameters denoted by a
    1806                 :            :  * different endpoint descriptor in usb_host_endpoint.
    1807                 :            :  * A call to xhci_add_endpoint() followed by a call to xhci_drop_endpoint() is
    1808                 :            :  * not allowed.
    1809                 :            :  *
    1810                 :            :  * The USB core will not allow URBs to be queued to an endpoint until the
    1811                 :            :  * configuration or alt setting is installed in the device, so there's no need
    1812                 :            :  * for mutual exclusion to protect the xhci->devs[slot_id] structure.
    1813                 :            :  */
    1814                 :          0 : static int xhci_add_endpoint(struct usb_hcd *hcd, struct usb_device *udev,
    1815                 :            :                 struct usb_host_endpoint *ep)
    1816                 :            : {
    1817                 :          0 :         struct xhci_hcd *xhci;
    1818                 :          0 :         struct xhci_container_ctx *in_ctx;
    1819                 :          0 :         unsigned int ep_index;
    1820                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    1821                 :          0 :         struct xhci_ep_ctx *ep_ctx;
    1822                 :          0 :         u32 added_ctxs;
    1823                 :          0 :         u32 new_add_flags, new_drop_flags;
    1824                 :          0 :         struct xhci_virt_device *virt_dev;
    1825                 :          0 :         int ret = 0;
    1826                 :            : 
    1827                 :          0 :         ret = xhci_check_args(hcd, udev, ep, 1, true, __func__);
    1828         [ #  # ]:          0 :         if (ret <= 0) {
    1829                 :            :                 /* So we won't queue a reset ep command for a root hub */
    1830                 :          0 :                 ep->hcpriv = NULL;
    1831                 :          0 :                 return ret;
    1832                 :            :         }
    1833                 :          0 :         xhci = hcd_to_xhci(hcd);
    1834         [ #  # ]:          0 :         if (xhci->xhc_state & XHCI_STATE_DYING)
    1835                 :            :                 return -ENODEV;
    1836                 :            : 
    1837         [ #  # ]:          0 :         added_ctxs = xhci_get_endpoint_flag(&ep->desc);
    1838         [ #  # ]:          0 :         if (added_ctxs == SLOT_FLAG || added_ctxs == EP0_FLAG) {
    1839                 :            :                 /* FIXME when we have to issue an evaluate endpoint command to
    1840                 :            :                  * deal with ep0 max packet size changing once we get the
    1841                 :            :                  * descriptors
    1842                 :            :                  */
    1843                 :            :                 xhci_dbg(xhci, "xHCI %s - can't add slot or ep 0 %#x\n",
    1844                 :            :                                 __func__, added_ctxs);
    1845                 :            :                 return 0;
    1846                 :            :         }
    1847                 :            : 
    1848                 :          0 :         virt_dev = xhci->devs[udev->slot_id];
    1849                 :          0 :         in_ctx = virt_dev->in_ctx;
    1850                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
    1851         [ #  # ]:          0 :         if (!ctrl_ctx) {
    1852                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    1853                 :            :                                 __func__);
    1854                 :          0 :                 return 0;
    1855                 :            :         }
    1856                 :            : 
    1857         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&ep->desc);
    1858                 :            :         /* If this endpoint is already in use, and the upper layers are trying
    1859                 :            :          * to add it again without dropping it, reject the addition.
    1860                 :            :          */
    1861         [ #  # ]:          0 :         if (virt_dev->eps[ep_index].ring &&
    1862         [ #  # ]:          0 :                         !(le32_to_cpu(ctrl_ctx->drop_flags) & added_ctxs)) {
    1863                 :          0 :                 xhci_warn(xhci, "Trying to add endpoint 0x%x "
    1864                 :            :                                 "without dropping it.\n",
    1865                 :            :                                 (unsigned int) ep->desc.bEndpointAddress);
    1866                 :          0 :                 return -EINVAL;
    1867                 :            :         }
    1868                 :            : 
    1869                 :            :         /* If the HCD has already noted the endpoint is enabled,
    1870                 :            :          * ignore this request.
    1871                 :            :          */
    1872         [ #  # ]:          0 :         if (le32_to_cpu(ctrl_ctx->add_flags) & added_ctxs) {
    1873                 :          0 :                 xhci_warn(xhci, "xHCI %s called with enabled ep %p\n",
    1874                 :            :                                 __func__, ep);
    1875                 :          0 :                 return 0;
    1876                 :            :         }
    1877                 :            : 
    1878                 :            :         /*
    1879                 :            :          * Configuration and alternate setting changes must be done in
    1880                 :            :          * process context, not interrupt context (or so documenation
    1881                 :            :          * for usb_set_interface() and usb_set_configuration() claim).
    1882                 :            :          */
    1883         [ #  # ]:          0 :         if (xhci_endpoint_init(xhci, virt_dev, udev, ep, GFP_NOIO) < 0) {
    1884                 :            :                 dev_dbg(&udev->dev, "%s - could not initialize ep %#x\n",
    1885                 :            :                                 __func__, ep->desc.bEndpointAddress);
    1886                 :            :                 return -ENOMEM;
    1887                 :            :         }
    1888                 :            : 
    1889                 :          0 :         if (xhci->quirks & XHCI_MTK_HOST) {
    1890                 :            :                 ret = xhci_mtk_add_ep_quirk(hcd, udev, ep);
    1891                 :            :                 if (ret < 0) {
    1892                 :            :                         xhci_ring_free(xhci, virt_dev->eps[ep_index].new_ring);
    1893                 :            :                         virt_dev->eps[ep_index].new_ring = NULL;
    1894                 :            :                         return ret;
    1895                 :            :                 }
    1896                 :            :         }
    1897                 :            : 
    1898                 :          0 :         ctrl_ctx->add_flags |= cpu_to_le32(added_ctxs);
    1899                 :          0 :         new_add_flags = le32_to_cpu(ctrl_ctx->add_flags);
    1900                 :            : 
    1901                 :            :         /* If xhci_endpoint_disable() was called for this endpoint, but the
    1902                 :            :          * xHC hasn't been notified yet through the check_bandwidth() call,
    1903                 :            :          * this re-adds a new state for the endpoint from the new endpoint
    1904                 :            :          * descriptors.  We must drop and re-add this endpoint, so we leave the
    1905                 :            :          * drop flags alone.
    1906                 :            :          */
    1907                 :          0 :         new_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags);
    1908                 :            : 
    1909                 :            :         /* Store the usb_device pointer for later use */
    1910                 :          0 :         ep->hcpriv = udev;
    1911                 :            : 
    1912                 :          0 :         ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, ep_index);
    1913                 :          0 :         trace_xhci_add_endpoint(ep_ctx);
    1914                 :            : 
    1915                 :          0 :         xhci_debugfs_create_endpoint(xhci, virt_dev, ep_index);
    1916                 :            : 
    1917                 :          0 :         xhci_dbg(xhci, "add ep 0x%x, slot id %d, new drop flags = %#x, new add flags = %#x\n",
    1918                 :            :                         (unsigned int) ep->desc.bEndpointAddress,
    1919                 :            :                         udev->slot_id,
    1920                 :            :                         (unsigned int) new_drop_flags,
    1921                 :            :                         (unsigned int) new_add_flags);
    1922                 :          0 :         return 0;
    1923                 :            : }
    1924                 :            : 
    1925                 :            : static void xhci_zero_in_ctx(struct xhci_hcd *xhci, struct xhci_virt_device *virt_dev)
    1926                 :            : {
    1927                 :            :         struct xhci_input_control_ctx *ctrl_ctx;
    1928                 :            :         struct xhci_ep_ctx *ep_ctx;
    1929                 :            :         struct xhci_slot_ctx *slot_ctx;
    1930                 :            :         int i;
    1931                 :            : 
    1932                 :            :         ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
    1933                 :            :         if (!ctrl_ctx) {
    1934                 :            :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    1935                 :            :                                 __func__);
    1936                 :            :                 return;
    1937                 :            :         }
    1938                 :            : 
    1939                 :            :         /* When a device's add flag and drop flag are zero, any subsequent
    1940                 :            :          * configure endpoint command will leave that endpoint's state
    1941                 :            :          * untouched.  Make sure we don't leave any old state in the input
    1942                 :            :          * endpoint contexts.
    1943                 :            :          */
    1944                 :            :         ctrl_ctx->drop_flags = 0;
    1945                 :            :         ctrl_ctx->add_flags = 0;
    1946                 :            :         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
    1947                 :            :         slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
    1948                 :            :         /* Endpoint 0 is always valid */
    1949                 :            :         slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(1));
    1950                 :            :         for (i = 1; i < 31; i++) {
    1951                 :            :                 ep_ctx = xhci_get_ep_ctx(xhci, virt_dev->in_ctx, i);
    1952                 :            :                 ep_ctx->ep_info = 0;
    1953                 :            :                 ep_ctx->ep_info2 = 0;
    1954                 :            :                 ep_ctx->deq = 0;
    1955                 :            :                 ep_ctx->tx_info = 0;
    1956                 :            :         }
    1957                 :            : }
    1958                 :            : 
    1959                 :            : static int xhci_configure_endpoint_result(struct xhci_hcd *xhci,
    1960                 :            :                 struct usb_device *udev, u32 *cmd_status)
    1961                 :            : {
    1962                 :            :         int ret;
    1963                 :            : 
    1964                 :            :         switch (*cmd_status) {
    1965                 :            :         case COMP_COMMAND_ABORTED:
    1966                 :            :         case COMP_COMMAND_RING_STOPPED:
    1967                 :            :                 xhci_warn(xhci, "Timeout while waiting for configure endpoint command\n");
    1968                 :            :                 ret = -ETIME;
    1969                 :            :                 break;
    1970                 :            :         case COMP_RESOURCE_ERROR:
    1971                 :            :                 dev_warn(&udev->dev,
    1972                 :            :                          "Not enough host controller resources for new device state.\n");
    1973                 :            :                 ret = -ENOMEM;
    1974                 :            :                 /* FIXME: can we allocate more resources for the HC? */
    1975                 :            :                 break;
    1976                 :            :         case COMP_BANDWIDTH_ERROR:
    1977                 :            :         case COMP_SECONDARY_BANDWIDTH_ERROR:
    1978                 :            :                 dev_warn(&udev->dev,
    1979                 :            :                          "Not enough bandwidth for new device state.\n");
    1980                 :            :                 ret = -ENOSPC;
    1981                 :            :                 /* FIXME: can we go back to the old state? */
    1982                 :            :                 break;
    1983                 :            :         case COMP_TRB_ERROR:
    1984                 :            :                 /* the HCD set up something wrong */
    1985                 :            :                 dev_warn(&udev->dev, "ERROR: Endpoint drop flag = 0, "
    1986                 :            :                                 "add flag = 1, "
    1987                 :            :                                 "and endpoint is not disabled.\n");
    1988                 :            :                 ret = -EINVAL;
    1989                 :            :                 break;
    1990                 :            :         case COMP_INCOMPATIBLE_DEVICE_ERROR:
    1991                 :            :                 dev_warn(&udev->dev,
    1992                 :            :                          "ERROR: Incompatible device for endpoint configure command.\n");
    1993                 :            :                 ret = -ENODEV;
    1994                 :            :                 break;
    1995                 :            :         case COMP_SUCCESS:
    1996                 :            :                 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
    1997                 :            :                                 "Successful Endpoint Configure command");
    1998                 :            :                 ret = 0;
    1999                 :            :                 break;
    2000                 :            :         default:
    2001                 :            :                 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
    2002                 :            :                                 *cmd_status);
    2003                 :            :                 ret = -EINVAL;
    2004                 :            :                 break;
    2005                 :            :         }
    2006                 :            :         return ret;
    2007                 :            : }
    2008                 :            : 
    2009                 :            : static int xhci_evaluate_context_result(struct xhci_hcd *xhci,
    2010                 :            :                 struct usb_device *udev, u32 *cmd_status)
    2011                 :            : {
    2012                 :            :         int ret;
    2013                 :            : 
    2014                 :            :         switch (*cmd_status) {
    2015                 :            :         case COMP_COMMAND_ABORTED:
    2016                 :            :         case COMP_COMMAND_RING_STOPPED:
    2017                 :            :                 xhci_warn(xhci, "Timeout while waiting for evaluate context command\n");
    2018                 :            :                 ret = -ETIME;
    2019                 :            :                 break;
    2020                 :            :         case COMP_PARAMETER_ERROR:
    2021                 :            :                 dev_warn(&udev->dev,
    2022                 :            :                          "WARN: xHCI driver setup invalid evaluate context command.\n");
    2023                 :            :                 ret = -EINVAL;
    2024                 :            :                 break;
    2025                 :            :         case COMP_SLOT_NOT_ENABLED_ERROR:
    2026                 :            :                 dev_warn(&udev->dev,
    2027                 :            :                         "WARN: slot not enabled for evaluate context command.\n");
    2028                 :            :                 ret = -EINVAL;
    2029                 :            :                 break;
    2030                 :            :         case COMP_CONTEXT_STATE_ERROR:
    2031                 :            :                 dev_warn(&udev->dev,
    2032                 :            :                         "WARN: invalid context state for evaluate context command.\n");
    2033                 :            :                 ret = -EINVAL;
    2034                 :            :                 break;
    2035                 :            :         case COMP_INCOMPATIBLE_DEVICE_ERROR:
    2036                 :            :                 dev_warn(&udev->dev,
    2037                 :            :                         "ERROR: Incompatible device for evaluate context command.\n");
    2038                 :            :                 ret = -ENODEV;
    2039                 :            :                 break;
    2040                 :            :         case COMP_MAX_EXIT_LATENCY_TOO_LARGE_ERROR:
    2041                 :            :                 /* Max Exit Latency too large error */
    2042                 :            :                 dev_warn(&udev->dev, "WARN: Max Exit Latency too large\n");
    2043                 :            :                 ret = -EINVAL;
    2044                 :            :                 break;
    2045                 :            :         case COMP_SUCCESS:
    2046                 :            :                 xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
    2047                 :            :                                 "Successful evaluate context command");
    2048                 :            :                 ret = 0;
    2049                 :            :                 break;
    2050                 :            :         default:
    2051                 :            :                 xhci_err(xhci, "ERROR: unexpected command completion code 0x%x.\n",
    2052                 :            :                         *cmd_status);
    2053                 :            :                 ret = -EINVAL;
    2054                 :            :                 break;
    2055                 :            :         }
    2056                 :            :         return ret;
    2057                 :            : }
    2058                 :            : 
    2059                 :            : static u32 xhci_count_num_new_endpoints(struct xhci_hcd *xhci,
    2060                 :            :                 struct xhci_input_control_ctx *ctrl_ctx)
    2061                 :            : {
    2062                 :            :         u32 valid_add_flags;
    2063                 :            :         u32 valid_drop_flags;
    2064                 :            : 
    2065                 :            :         /* Ignore the slot flag (bit 0), and the default control endpoint flag
    2066                 :            :          * (bit 1).  The default control endpoint is added during the Address
    2067                 :            :          * Device command and is never removed until the slot is disabled.
    2068                 :            :          */
    2069                 :            :         valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
    2070                 :            :         valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
    2071                 :            : 
    2072                 :            :         /* Use hweight32 to count the number of ones in the add flags, or
    2073                 :            :          * number of endpoints added.  Don't count endpoints that are changed
    2074                 :            :          * (both added and dropped).
    2075                 :            :          */
    2076                 :            :         return hweight32(valid_add_flags) -
    2077                 :            :                 hweight32(valid_add_flags & valid_drop_flags);
    2078                 :            : }
    2079                 :            : 
    2080                 :            : static unsigned int xhci_count_num_dropped_endpoints(struct xhci_hcd *xhci,
    2081                 :            :                 struct xhci_input_control_ctx *ctrl_ctx)
    2082                 :            : {
    2083                 :            :         u32 valid_add_flags;
    2084                 :            :         u32 valid_drop_flags;
    2085                 :            : 
    2086                 :            :         valid_add_flags = le32_to_cpu(ctrl_ctx->add_flags) >> 2;
    2087                 :            :         valid_drop_flags = le32_to_cpu(ctrl_ctx->drop_flags) >> 2;
    2088                 :            : 
    2089                 :            :         return hweight32(valid_drop_flags) -
    2090                 :            :                 hweight32(valid_add_flags & valid_drop_flags);
    2091                 :            : }
    2092                 :            : 
    2093                 :            : /*
    2094                 :            :  * We need to reserve the new number of endpoints before the configure endpoint
    2095                 :            :  * command completes.  We can't subtract the dropped endpoints from the number
    2096                 :            :  * of active endpoints until the command completes because we can oversubscribe
    2097                 :            :  * the host in this case:
    2098                 :            :  *
    2099                 :            :  *  - the first configure endpoint command drops more endpoints than it adds
    2100                 :            :  *  - a second configure endpoint command that adds more endpoints is queued
    2101                 :            :  *  - the first configure endpoint command fails, so the config is unchanged
    2102                 :            :  *  - the second command may succeed, even though there isn't enough resources
    2103                 :            :  *
    2104                 :            :  * Must be called with xhci->lock held.
    2105                 :            :  */
    2106                 :          0 : static int xhci_reserve_host_resources(struct xhci_hcd *xhci,
    2107                 :            :                 struct xhci_input_control_ctx *ctrl_ctx)
    2108                 :            : {
    2109                 :          0 :         u32 added_eps;
    2110                 :            : 
    2111                 :          0 :         added_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
    2112         [ #  # ]:          0 :         if (xhci->num_active_eps + added_eps > xhci->limit_active_eps) {
    2113                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2114                 :            :                                 "Not enough ep ctxs: "
    2115                 :            :                                 "%u active, need to add %u, limit is %u.",
    2116                 :            :                                 xhci->num_active_eps, added_eps,
    2117                 :            :                                 xhci->limit_active_eps);
    2118                 :          0 :                 return -ENOMEM;
    2119                 :            :         }
    2120                 :          0 :         xhci->num_active_eps += added_eps;
    2121                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2122                 :            :                         "Adding %u ep ctxs, %u now active.", added_eps,
    2123                 :            :                         xhci->num_active_eps);
    2124                 :          0 :         return 0;
    2125                 :            : }
    2126                 :            : 
    2127                 :            : /*
    2128                 :            :  * The configure endpoint was failed by the xHC for some other reason, so we
    2129                 :            :  * need to revert the resources that failed configuration would have used.
    2130                 :            :  *
    2131                 :            :  * Must be called with xhci->lock held.
    2132                 :            :  */
    2133                 :          0 : static void xhci_free_host_resources(struct xhci_hcd *xhci,
    2134                 :            :                 struct xhci_input_control_ctx *ctrl_ctx)
    2135                 :            : {
    2136                 :          0 :         u32 num_failed_eps;
    2137                 :            : 
    2138                 :          0 :         num_failed_eps = xhci_count_num_new_endpoints(xhci, ctrl_ctx);
    2139                 :          0 :         xhci->num_active_eps -= num_failed_eps;
    2140                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2141                 :            :                         "Removing %u failed ep ctxs, %u now active.",
    2142                 :            :                         num_failed_eps,
    2143                 :            :                         xhci->num_active_eps);
    2144                 :          0 : }
    2145                 :            : 
    2146                 :            : /*
    2147                 :            :  * Now that the command has completed, clean up the active endpoint count by
    2148                 :            :  * subtracting out the endpoints that were dropped (but not changed).
    2149                 :            :  *
    2150                 :            :  * Must be called with xhci->lock held.
    2151                 :            :  */
    2152                 :          0 : static void xhci_finish_resource_reservation(struct xhci_hcd *xhci,
    2153                 :            :                 struct xhci_input_control_ctx *ctrl_ctx)
    2154                 :            : {
    2155                 :          0 :         u32 num_dropped_eps;
    2156                 :            : 
    2157                 :          0 :         num_dropped_eps = xhci_count_num_dropped_endpoints(xhci, ctrl_ctx);
    2158                 :          0 :         xhci->num_active_eps -= num_dropped_eps;
    2159         [ #  # ]:          0 :         if (num_dropped_eps)
    2160                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2161                 :            :                                 "Removing %u dropped ep ctxs, %u now active.",
    2162                 :            :                                 num_dropped_eps,
    2163                 :            :                                 xhci->num_active_eps);
    2164                 :          0 : }
    2165                 :            : 
    2166                 :          0 : static unsigned int xhci_get_block_size(struct usb_device *udev)
    2167                 :            : {
    2168                 :          0 :         switch (udev->speed) {
    2169                 :            :         case USB_SPEED_LOW:
    2170                 :            :         case USB_SPEED_FULL:
    2171                 :            :                 return FS_BLOCK;
    2172                 :          0 :         case USB_SPEED_HIGH:
    2173                 :          0 :                 return HS_BLOCK;
    2174                 :            :         case USB_SPEED_SUPER:
    2175                 :            :         case USB_SPEED_SUPER_PLUS:
    2176                 :            :                 return SS_BLOCK;
    2177                 :            :         case USB_SPEED_UNKNOWN:
    2178                 :            :         case USB_SPEED_WIRELESS:
    2179                 :            :         default:
    2180                 :            :                 /* Should never happen */
    2181                 :            :                 return 1;
    2182                 :            :         }
    2183                 :            : }
    2184                 :            : 
    2185                 :            : static unsigned int
    2186                 :          0 : xhci_get_largest_overhead(struct xhci_interval_bw *interval_bw)
    2187                 :            : {
    2188                 :          0 :         if (interval_bw->overhead[LS_OVERHEAD_TYPE])
    2189                 :            :                 return LS_OVERHEAD;
    2190   [ #  #  #  # ]:          0 :         if (interval_bw->overhead[FS_OVERHEAD_TYPE])
    2191                 :          0 :                 return FS_OVERHEAD;
    2192                 :            :         return HS_OVERHEAD;
    2193                 :            : }
    2194                 :            : 
    2195                 :            : /* If we are changing a LS/FS device under a HS hub,
    2196                 :            :  * make sure (if we are activating a new TT) that the HS bus has enough
    2197                 :            :  * bandwidth for this new TT.
    2198                 :            :  */
    2199                 :          0 : static int xhci_check_tt_bw_table(struct xhci_hcd *xhci,
    2200                 :            :                 struct xhci_virt_device *virt_dev,
    2201                 :            :                 int old_active_eps)
    2202                 :            : {
    2203                 :          0 :         struct xhci_interval_bw_table *bw_table;
    2204                 :          0 :         struct xhci_tt_bw_info *tt_info;
    2205                 :            : 
    2206                 :            :         /* Find the bandwidth table for the root port this TT is attached to. */
    2207                 :          0 :         bw_table = &xhci->rh_bw[virt_dev->real_port - 1].bw_table;
    2208                 :          0 :         tt_info = virt_dev->tt_info;
    2209                 :            :         /* If this TT already had active endpoints, the bandwidth for this TT
    2210                 :            :          * has already been added.  Removing all periodic endpoints (and thus
    2211                 :            :          * making the TT enactive) will only decrease the bandwidth used.
    2212                 :            :          */
    2213                 :          0 :         if (old_active_eps)
    2214                 :            :                 return 0;
    2215         [ #  # ]:          0 :         if (old_active_eps == 0 && tt_info->active_eps != 0) {
    2216         [ #  # ]:          0 :                 if (bw_table->bw_used + TT_HS_OVERHEAD > HS_BW_LIMIT)
    2217                 :            :                         return -ENOMEM;
    2218                 :            :                 return 0;
    2219                 :            :         }
    2220                 :            :         /* Not sure why we would have no new active endpoints...
    2221                 :            :          *
    2222                 :            :          * Maybe because of an Evaluate Context change for a hub update or a
    2223                 :            :          * control endpoint 0 max packet size change?
    2224                 :            :          * FIXME: skip the bandwidth calculation in that case.
    2225                 :            :          */
    2226                 :            :         return 0;
    2227                 :            : }
    2228                 :            : 
    2229                 :          0 : static int xhci_check_ss_bw(struct xhci_hcd *xhci,
    2230                 :            :                 struct xhci_virt_device *virt_dev)
    2231                 :            : {
    2232                 :          0 :         unsigned int bw_reserved;
    2233                 :            : 
    2234                 :          0 :         bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_IN, 100);
    2235                 :          0 :         if (virt_dev->bw_table->ss_bw_in > (SS_BW_LIMIT_IN - bw_reserved))
    2236                 :            :                 return -ENOMEM;
    2237                 :            : 
    2238                 :          0 :         bw_reserved = DIV_ROUND_UP(SS_BW_RESERVED*SS_BW_LIMIT_OUT, 100);
    2239         [ #  # ]:          0 :         if (virt_dev->bw_table->ss_bw_out > (SS_BW_LIMIT_OUT - bw_reserved))
    2240                 :          0 :                 return -ENOMEM;
    2241                 :            : 
    2242                 :            :         return 0;
    2243                 :            : }
    2244                 :            : 
    2245                 :            : /*
    2246                 :            :  * This algorithm is a very conservative estimate of the worst-case scheduling
    2247                 :            :  * scenario for any one interval.  The hardware dynamically schedules the
    2248                 :            :  * packets, so we can't tell which microframe could be the limiting factor in
    2249                 :            :  * the bandwidth scheduling.  This only takes into account periodic endpoints.
    2250                 :            :  *
    2251                 :            :  * Obviously, we can't solve an NP complete problem to find the minimum worst
    2252                 :            :  * case scenario.  Instead, we come up with an estimate that is no less than
    2253                 :            :  * the worst case bandwidth used for any one microframe, but may be an
    2254                 :            :  * over-estimate.
    2255                 :            :  *
    2256                 :            :  * We walk the requirements for each endpoint by interval, starting with the
    2257                 :            :  * smallest interval, and place packets in the schedule where there is only one
    2258                 :            :  * possible way to schedule packets for that interval.  In order to simplify
    2259                 :            :  * this algorithm, we record the largest max packet size for each interval, and
    2260                 :            :  * assume all packets will be that size.
    2261                 :            :  *
    2262                 :            :  * For interval 0, we obviously must schedule all packets for each interval.
    2263                 :            :  * The bandwidth for interval 0 is just the amount of data to be transmitted
    2264                 :            :  * (the sum of all max ESIT payload sizes, plus any overhead per packet times
    2265                 :            :  * the number of packets).
    2266                 :            :  *
    2267                 :            :  * For interval 1, we have two possible microframes to schedule those packets
    2268                 :            :  * in.  For this algorithm, if we can schedule the same number of packets for
    2269                 :            :  * each possible scheduling opportunity (each microframe), we will do so.  The
    2270                 :            :  * remaining number of packets will be saved to be transmitted in the gaps in
    2271                 :            :  * the next interval's scheduling sequence.
    2272                 :            :  *
    2273                 :            :  * As we move those remaining packets to be scheduled with interval 2 packets,
    2274                 :            :  * we have to double the number of remaining packets to transmit.  This is
    2275                 :            :  * because the intervals are actually powers of 2, and we would be transmitting
    2276                 :            :  * the previous interval's packets twice in this interval.  We also have to be
    2277                 :            :  * sure that when we look at the largest max packet size for this interval, we
    2278                 :            :  * also look at the largest max packet size for the remaining packets and take
    2279                 :            :  * the greater of the two.
    2280                 :            :  *
    2281                 :            :  * The algorithm continues to evenly distribute packets in each scheduling
    2282                 :            :  * opportunity, and push the remaining packets out, until we get to the last
    2283                 :            :  * interval.  Then those packets and their associated overhead are just added
    2284                 :            :  * to the bandwidth used.
    2285                 :            :  */
    2286                 :          0 : static int xhci_check_bw_table(struct xhci_hcd *xhci,
    2287                 :            :                 struct xhci_virt_device *virt_dev,
    2288                 :            :                 int old_active_eps)
    2289                 :            : {
    2290                 :          0 :         unsigned int bw_reserved;
    2291                 :          0 :         unsigned int max_bandwidth;
    2292                 :          0 :         unsigned int bw_used;
    2293                 :          0 :         unsigned int block_size;
    2294                 :          0 :         struct xhci_interval_bw_table *bw_table;
    2295                 :          0 :         unsigned int packet_size = 0;
    2296                 :          0 :         unsigned int overhead = 0;
    2297                 :          0 :         unsigned int packets_transmitted = 0;
    2298                 :          0 :         unsigned int packets_remaining = 0;
    2299                 :          0 :         unsigned int i;
    2300                 :            : 
    2301         [ #  # ]:          0 :         if (virt_dev->udev->speed >= USB_SPEED_SUPER)
    2302         [ #  # ]:          0 :                 return xhci_check_ss_bw(xhci, virt_dev);
    2303                 :            : 
    2304         [ #  # ]:          0 :         if (virt_dev->udev->speed == USB_SPEED_HIGH) {
    2305                 :            :                 max_bandwidth = HS_BW_LIMIT;
    2306                 :            :                 /* Convert percent of bus BW reserved to blocks reserved */
    2307                 :            :                 bw_reserved = DIV_ROUND_UP(HS_BW_RESERVED * max_bandwidth, 100);
    2308                 :            :         } else {
    2309                 :          0 :                 max_bandwidth = FS_BW_LIMIT;
    2310                 :          0 :                 bw_reserved = DIV_ROUND_UP(FS_BW_RESERVED * max_bandwidth, 100);
    2311                 :            :         }
    2312                 :            : 
    2313                 :          0 :         bw_table = virt_dev->bw_table;
    2314                 :            :         /* We need to translate the max packet size and max ESIT payloads into
    2315                 :            :          * the units the hardware uses.
    2316                 :            :          */
    2317         [ #  # ]:          0 :         block_size = xhci_get_block_size(virt_dev->udev);
    2318                 :            : 
    2319                 :            :         /* If we are manipulating a LS/FS device under a HS hub, double check
    2320                 :            :          * that the HS bus has enough bandwidth if we are activing a new TT.
    2321                 :            :          */
    2322         [ #  # ]:          0 :         if (virt_dev->tt_info) {
    2323                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2324                 :            :                                 "Recalculating BW for rootport %u",
    2325                 :          0 :                                 virt_dev->real_port);
    2326         [ #  # ]:          0 :                 if (xhci_check_tt_bw_table(xhci, virt_dev, old_active_eps)) {
    2327                 :          0 :                         xhci_warn(xhci, "Not enough bandwidth on HS bus for "
    2328                 :            :                                         "newly activated TT.\n");
    2329                 :          0 :                         return -ENOMEM;
    2330                 :            :                 }
    2331                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2332                 :            :                                 "Recalculating BW for TT slot %u port %u",
    2333                 :            :                                 virt_dev->tt_info->slot_id,
    2334                 :            :                                 virt_dev->tt_info->ttport);
    2335                 :            :         } else {
    2336                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2337                 :            :                                 "Recalculating BW for rootport %u",
    2338                 :          0 :                                 virt_dev->real_port);
    2339                 :            :         }
    2340                 :            : 
    2341                 :            :         /* Add in how much bandwidth will be used for interval zero, or the
    2342                 :            :          * rounded max ESIT payload + number of packets * largest overhead.
    2343                 :            :          */
    2344                 :          0 :         bw_used = DIV_ROUND_UP(bw_table->interval0_esit_payload, block_size) +
    2345                 :          0 :                 bw_table->interval_bw[0].num_packets *
    2346         [ #  # ]:          0 :                 xhci_get_largest_overhead(&bw_table->interval_bw[0]);
    2347                 :            : 
    2348         [ #  # ]:          0 :         for (i = 1; i < XHCI_MAX_INTERVAL; i++) {
    2349                 :          0 :                 unsigned int bw_added;
    2350                 :          0 :                 unsigned int largest_mps;
    2351                 :          0 :                 unsigned int interval_overhead;
    2352                 :            : 
    2353                 :            :                 /*
    2354                 :            :                  * How many packets could we transmit in this interval?
    2355                 :            :                  * If packets didn't fit in the previous interval, we will need
    2356                 :            :                  * to transmit that many packets twice within this interval.
    2357                 :            :                  */
    2358                 :          0 :                 packets_remaining = 2 * packets_remaining +
    2359                 :          0 :                         bw_table->interval_bw[i].num_packets;
    2360                 :            : 
    2361                 :            :                 /* Find the largest max packet size of this or the previous
    2362                 :            :                  * interval.
    2363                 :            :                  */
    2364         [ #  # ]:          0 :                 if (list_empty(&bw_table->interval_bw[i].endpoints))
    2365                 :            :                         largest_mps = 0;
    2366                 :            :                 else {
    2367                 :          0 :                         struct xhci_virt_ep *virt_ep;
    2368                 :          0 :                         struct list_head *ep_entry;
    2369                 :            : 
    2370                 :          0 :                         ep_entry = bw_table->interval_bw[i].endpoints.next;
    2371                 :          0 :                         virt_ep = list_entry(ep_entry,
    2372                 :            :                                         struct xhci_virt_ep, bw_endpoint_list);
    2373                 :            :                         /* Convert to blocks, rounding up */
    2374                 :          0 :                         largest_mps = DIV_ROUND_UP(
    2375                 :            :                                         virt_ep->bw_info.max_packet_size,
    2376                 :            :                                         block_size);
    2377                 :            :                 }
    2378                 :          0 :                 if (largest_mps > packet_size)
    2379                 :            :                         packet_size = largest_mps;
    2380                 :            : 
    2381                 :            :                 /* Use the larger overhead of this or the previous interval. */
    2382         [ #  # ]:          0 :                 interval_overhead = xhci_get_largest_overhead(
    2383                 :            :                                 &bw_table->interval_bw[i]);
    2384                 :          0 :                 if (interval_overhead > overhead)
    2385                 :            :                         overhead = interval_overhead;
    2386                 :            : 
    2387                 :            :                 /* How many packets can we evenly distribute across
    2388                 :            :                  * (1 << (i + 1)) possible scheduling opportunities?
    2389                 :            :                  */
    2390                 :          0 :                 packets_transmitted = packets_remaining >> (i + 1);
    2391                 :            : 
    2392                 :            :                 /* Add in the bandwidth used for those scheduled packets */
    2393                 :          0 :                 bw_added = packets_transmitted * (overhead + packet_size);
    2394                 :            : 
    2395                 :            :                 /* How many packets do we have remaining to transmit? */
    2396                 :          0 :                 packets_remaining = packets_remaining % (1 << (i + 1));
    2397                 :            : 
    2398                 :            :                 /* What largest max packet size should those packets have? */
    2399                 :            :                 /* If we've transmitted all packets, don't carry over the
    2400                 :            :                  * largest packet size.
    2401                 :            :                  */
    2402         [ #  # ]:          0 :                 if (packets_remaining == 0) {
    2403                 :            :                         packet_size = 0;
    2404                 :            :                         overhead = 0;
    2405         [ #  # ]:          0 :                 } else if (packets_transmitted > 0) {
    2406                 :            :                         /* Otherwise if we do have remaining packets, and we've
    2407                 :            :                          * scheduled some packets in this interval, take the
    2408                 :            :                          * largest max packet size from endpoints with this
    2409                 :            :                          * interval.
    2410                 :            :                          */
    2411                 :          0 :                         packet_size = largest_mps;
    2412                 :          0 :                         overhead = interval_overhead;
    2413                 :            :                 }
    2414                 :            :                 /* Otherwise carry over packet_size and overhead from the last
    2415                 :            :                  * time we had a remainder.
    2416                 :            :                  */
    2417                 :          0 :                 bw_used += bw_added;
    2418         [ #  # ]:          0 :                 if (bw_used > max_bandwidth) {
    2419                 :          0 :                         xhci_warn(xhci, "Not enough bandwidth. "
    2420                 :            :                                         "Proposed: %u, Max: %u\n",
    2421                 :            :                                 bw_used, max_bandwidth);
    2422                 :          0 :                         return -ENOMEM;
    2423                 :            :                 }
    2424                 :            :         }
    2425                 :            :         /*
    2426                 :            :          * Ok, we know we have some packets left over after even-handedly
    2427                 :            :          * scheduling interval 15.  We don't know which microframes they will
    2428                 :            :          * fit into, so we over-schedule and say they will be scheduled every
    2429                 :            :          * microframe.
    2430                 :            :          */
    2431         [ #  # ]:          0 :         if (packets_remaining > 0)
    2432                 :          0 :                 bw_used += overhead + packet_size;
    2433                 :            : 
    2434   [ #  #  #  # ]:          0 :         if (!virt_dev->tt_info && virt_dev->udev->speed == USB_SPEED_HIGH) {
    2435                 :          0 :                 unsigned int port_index = virt_dev->real_port - 1;
    2436                 :            : 
    2437                 :            :                 /* OK, we're manipulating a HS device attached to a
    2438                 :            :                  * root port bandwidth domain.  Include the number of active TTs
    2439                 :            :                  * in the bandwidth used.
    2440                 :            :                  */
    2441                 :          0 :                 bw_used += TT_HS_OVERHEAD *
    2442                 :          0 :                         xhci->rh_bw[port_index].num_active_tts;
    2443                 :            :         }
    2444                 :            : 
    2445                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    2446                 :            :                 "Final bandwidth: %u, Limit: %u, Reserved: %u, "
    2447                 :            :                 "Available: %u " "percent",
    2448                 :            :                 bw_used, max_bandwidth, bw_reserved,
    2449                 :          0 :                 (max_bandwidth - bw_used - bw_reserved) * 100 /
    2450                 :            :                 max_bandwidth);
    2451                 :            : 
    2452                 :          0 :         bw_used += bw_reserved;
    2453         [ #  # ]:          0 :         if (bw_used > max_bandwidth) {
    2454                 :          0 :                 xhci_warn(xhci, "Not enough bandwidth. Proposed: %u, Max: %u\n",
    2455                 :            :                                 bw_used, max_bandwidth);
    2456                 :          0 :                 return -ENOMEM;
    2457                 :            :         }
    2458                 :            : 
    2459                 :          0 :         bw_table->bw_used = bw_used;
    2460                 :          0 :         return 0;
    2461                 :            : }
    2462                 :            : 
    2463                 :            : static bool xhci_is_async_ep(unsigned int ep_type)
    2464                 :            : {
    2465                 :            :         return (ep_type != ISOC_OUT_EP && ep_type != INT_OUT_EP &&
    2466                 :            :                                         ep_type != ISOC_IN_EP &&
    2467                 :            :                                         ep_type != INT_IN_EP);
    2468                 :            : }
    2469                 :            : 
    2470                 :            : static bool xhci_is_sync_in_ep(unsigned int ep_type)
    2471                 :            : {
    2472                 :            :         return (ep_type == ISOC_IN_EP || ep_type == INT_IN_EP);
    2473                 :            : }
    2474                 :            : 
    2475                 :          0 : static unsigned int xhci_get_ss_bw_consumed(struct xhci_bw_info *ep_bw)
    2476                 :            : {
    2477                 :          0 :         unsigned int mps = DIV_ROUND_UP(ep_bw->max_packet_size, SS_BLOCK);
    2478                 :            : 
    2479         [ #  # ]:          0 :         if (ep_bw->ep_interval == 0)
    2480                 :          0 :                 return SS_OVERHEAD_BURST +
    2481                 :          0 :                         (ep_bw->mult * ep_bw->num_packets *
    2482                 :          0 :                                         (SS_OVERHEAD + mps));
    2483                 :          0 :         return DIV_ROUND_UP(ep_bw->mult * ep_bw->num_packets *
    2484                 :            :                                 (SS_OVERHEAD + mps + SS_OVERHEAD_BURST),
    2485                 :            :                                 1 << ep_bw->ep_interval);
    2486                 :            : 
    2487                 :            : }
    2488                 :            : 
    2489                 :            : static void xhci_drop_ep_from_interval_table(struct xhci_hcd *xhci,
    2490                 :            :                 struct xhci_bw_info *ep_bw,
    2491                 :            :                 struct xhci_interval_bw_table *bw_table,
    2492                 :            :                 struct usb_device *udev,
    2493                 :            :                 struct xhci_virt_ep *virt_ep,
    2494                 :            :                 struct xhci_tt_bw_info *tt_info)
    2495                 :            : {
    2496                 :            :         struct xhci_interval_bw *interval_bw;
    2497                 :            :         int normalized_interval;
    2498                 :            : 
    2499                 :            :         if (xhci_is_async_ep(ep_bw->type))
    2500                 :            :                 return;
    2501                 :            : 
    2502                 :            :         if (udev->speed >= USB_SPEED_SUPER) {
    2503                 :            :                 if (xhci_is_sync_in_ep(ep_bw->type))
    2504                 :            :                         xhci->devs[udev->slot_id]->bw_table->ss_bw_in -=
    2505                 :            :                                 xhci_get_ss_bw_consumed(ep_bw);
    2506                 :            :                 else
    2507                 :            :                         xhci->devs[udev->slot_id]->bw_table->ss_bw_out -=
    2508                 :            :                                 xhci_get_ss_bw_consumed(ep_bw);
    2509                 :            :                 return;
    2510                 :            :         }
    2511                 :            : 
    2512                 :            :         /* SuperSpeed endpoints never get added to intervals in the table, so
    2513                 :            :          * this check is only valid for HS/FS/LS devices.
    2514                 :            :          */
    2515                 :            :         if (list_empty(&virt_ep->bw_endpoint_list))
    2516                 :            :                 return;
    2517                 :            :         /* For LS/FS devices, we need to translate the interval expressed in
    2518                 :            :          * microframes to frames.
    2519                 :            :          */
    2520                 :            :         if (udev->speed == USB_SPEED_HIGH)
    2521                 :            :                 normalized_interval = ep_bw->ep_interval;
    2522                 :            :         else
    2523                 :            :                 normalized_interval = ep_bw->ep_interval - 3;
    2524                 :            : 
    2525                 :            :         if (normalized_interval == 0)
    2526                 :            :                 bw_table->interval0_esit_payload -= ep_bw->max_esit_payload;
    2527                 :            :         interval_bw = &bw_table->interval_bw[normalized_interval];
    2528                 :            :         interval_bw->num_packets -= ep_bw->num_packets;
    2529                 :            :         switch (udev->speed) {
    2530                 :            :         case USB_SPEED_LOW:
    2531                 :            :                 interval_bw->overhead[LS_OVERHEAD_TYPE] -= 1;
    2532                 :            :                 break;
    2533                 :            :         case USB_SPEED_FULL:
    2534                 :            :                 interval_bw->overhead[FS_OVERHEAD_TYPE] -= 1;
    2535                 :            :                 break;
    2536                 :            :         case USB_SPEED_HIGH:
    2537                 :            :                 interval_bw->overhead[HS_OVERHEAD_TYPE] -= 1;
    2538                 :            :                 break;
    2539                 :            :         case USB_SPEED_SUPER:
    2540                 :            :         case USB_SPEED_SUPER_PLUS:
    2541                 :            :         case USB_SPEED_UNKNOWN:
    2542                 :            :         case USB_SPEED_WIRELESS:
    2543                 :            :                 /* Should never happen because only LS/FS/HS endpoints will get
    2544                 :            :                  * added to the endpoint list.
    2545                 :            :                  */
    2546                 :            :                 return;
    2547                 :            :         }
    2548                 :            :         if (tt_info)
    2549                 :            :                 tt_info->active_eps -= 1;
    2550                 :            :         list_del_init(&virt_ep->bw_endpoint_list);
    2551                 :            : }
    2552                 :            : 
    2553                 :            : static void xhci_add_ep_to_interval_table(struct xhci_hcd *xhci,
    2554                 :            :                 struct xhci_bw_info *ep_bw,
    2555                 :            :                 struct xhci_interval_bw_table *bw_table,
    2556                 :            :                 struct usb_device *udev,
    2557                 :            :                 struct xhci_virt_ep *virt_ep,
    2558                 :            :                 struct xhci_tt_bw_info *tt_info)
    2559                 :            : {
    2560                 :            :         struct xhci_interval_bw *interval_bw;
    2561                 :            :         struct xhci_virt_ep *smaller_ep;
    2562                 :            :         int normalized_interval;
    2563                 :            : 
    2564                 :            :         if (xhci_is_async_ep(ep_bw->type))
    2565                 :            :                 return;
    2566                 :            : 
    2567                 :            :         if (udev->speed == USB_SPEED_SUPER) {
    2568                 :            :                 if (xhci_is_sync_in_ep(ep_bw->type))
    2569                 :            :                         xhci->devs[udev->slot_id]->bw_table->ss_bw_in +=
    2570                 :            :                                 xhci_get_ss_bw_consumed(ep_bw);
    2571                 :            :                 else
    2572                 :            :                         xhci->devs[udev->slot_id]->bw_table->ss_bw_out +=
    2573                 :            :                                 xhci_get_ss_bw_consumed(ep_bw);
    2574                 :            :                 return;
    2575                 :            :         }
    2576                 :            : 
    2577                 :            :         /* For LS/FS devices, we need to translate the interval expressed in
    2578                 :            :          * microframes to frames.
    2579                 :            :          */
    2580                 :            :         if (udev->speed == USB_SPEED_HIGH)
    2581                 :            :                 normalized_interval = ep_bw->ep_interval;
    2582                 :            :         else
    2583                 :            :                 normalized_interval = ep_bw->ep_interval - 3;
    2584                 :            : 
    2585                 :            :         if (normalized_interval == 0)
    2586                 :            :                 bw_table->interval0_esit_payload += ep_bw->max_esit_payload;
    2587                 :            :         interval_bw = &bw_table->interval_bw[normalized_interval];
    2588                 :            :         interval_bw->num_packets += ep_bw->num_packets;
    2589                 :            :         switch (udev->speed) {
    2590                 :            :         case USB_SPEED_LOW:
    2591                 :            :                 interval_bw->overhead[LS_OVERHEAD_TYPE] += 1;
    2592                 :            :                 break;
    2593                 :            :         case USB_SPEED_FULL:
    2594                 :            :                 interval_bw->overhead[FS_OVERHEAD_TYPE] += 1;
    2595                 :            :                 break;
    2596                 :            :         case USB_SPEED_HIGH:
    2597                 :            :                 interval_bw->overhead[HS_OVERHEAD_TYPE] += 1;
    2598                 :            :                 break;
    2599                 :            :         case USB_SPEED_SUPER:
    2600                 :            :         case USB_SPEED_SUPER_PLUS:
    2601                 :            :         case USB_SPEED_UNKNOWN:
    2602                 :            :         case USB_SPEED_WIRELESS:
    2603                 :            :                 /* Should never happen because only LS/FS/HS endpoints will get
    2604                 :            :                  * added to the endpoint list.
    2605                 :            :                  */
    2606                 :            :                 return;
    2607                 :            :         }
    2608                 :            : 
    2609                 :            :         if (tt_info)
    2610                 :            :                 tt_info->active_eps += 1;
    2611                 :            :         /* Insert the endpoint into the list, largest max packet size first. */
    2612                 :            :         list_for_each_entry(smaller_ep, &interval_bw->endpoints,
    2613                 :            :                         bw_endpoint_list) {
    2614                 :            :                 if (ep_bw->max_packet_size >=
    2615                 :            :                                 smaller_ep->bw_info.max_packet_size) {
    2616                 :            :                         /* Add the new ep before the smaller endpoint */
    2617                 :            :                         list_add_tail(&virt_ep->bw_endpoint_list,
    2618                 :            :                                         &smaller_ep->bw_endpoint_list);
    2619                 :            :                         return;
    2620                 :            :                 }
    2621                 :            :         }
    2622                 :            :         /* Add the new endpoint at the end of the list. */
    2623                 :            :         list_add_tail(&virt_ep->bw_endpoint_list,
    2624                 :            :                         &interval_bw->endpoints);
    2625                 :            : }
    2626                 :            : 
    2627                 :          0 : void xhci_update_tt_active_eps(struct xhci_hcd *xhci,
    2628                 :            :                 struct xhci_virt_device *virt_dev,
    2629                 :            :                 int old_active_eps)
    2630                 :            : {
    2631                 :          0 :         struct xhci_root_port_bw_info *rh_bw_info;
    2632         [ #  # ]:          0 :         if (!virt_dev->tt_info)
    2633                 :            :                 return;
    2634                 :            : 
    2635                 :          0 :         rh_bw_info = &xhci->rh_bw[virt_dev->real_port - 1];
    2636         [ #  # ]:          0 :         if (old_active_eps == 0 &&
    2637         [ #  # ]:          0 :                                 virt_dev->tt_info->active_eps != 0) {
    2638                 :          0 :                 rh_bw_info->num_active_tts += 1;
    2639                 :          0 :                 rh_bw_info->bw_table.bw_used += TT_HS_OVERHEAD;
    2640         [ #  # ]:          0 :         } else if (old_active_eps != 0 &&
    2641         [ #  # ]:          0 :                                 virt_dev->tt_info->active_eps == 0) {
    2642                 :          0 :                 rh_bw_info->num_active_tts -= 1;
    2643                 :          0 :                 rh_bw_info->bw_table.bw_used -= TT_HS_OVERHEAD;
    2644                 :            :         }
    2645                 :            : }
    2646                 :            : 
    2647                 :          0 : static int xhci_reserve_bandwidth(struct xhci_hcd *xhci,
    2648                 :            :                 struct xhci_virt_device *virt_dev,
    2649                 :            :                 struct xhci_container_ctx *in_ctx)
    2650                 :            : {
    2651                 :          0 :         struct xhci_bw_info ep_bw_info[31];
    2652                 :          0 :         int i;
    2653                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    2654                 :          0 :         int old_active_eps = 0;
    2655                 :            : 
    2656         [ #  # ]:          0 :         if (virt_dev->tt_info)
    2657                 :          0 :                 old_active_eps = virt_dev->tt_info->active_eps;
    2658                 :            : 
    2659                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
    2660         [ #  # ]:          0 :         if (!ctrl_ctx) {
    2661                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    2662                 :            :                                 __func__);
    2663                 :          0 :                 return -ENOMEM;
    2664                 :            :         }
    2665                 :            : 
    2666         [ #  # ]:          0 :         for (i = 0; i < 31; i++) {
    2667   [ #  #  #  # ]:          0 :                 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
    2668                 :          0 :                         continue;
    2669                 :            : 
    2670                 :            :                 /* Make a copy of the BW info in case we need to revert this */
    2671                 :          0 :                 memcpy(&ep_bw_info[i], &virt_dev->eps[i].bw_info,
    2672                 :            :                                 sizeof(ep_bw_info[i]));
    2673                 :            :                 /* Drop the endpoint from the interval table if the endpoint is
    2674                 :            :                  * being dropped or changed.
    2675                 :            :                  */
    2676         [ #  # ]:          0 :                 if (EP_IS_DROPPED(ctrl_ctx, i))
    2677                 :          0 :                         xhci_drop_ep_from_interval_table(xhci,
    2678                 :            :                                         &virt_dev->eps[i].bw_info,
    2679                 :            :                                         virt_dev->bw_table,
    2680                 :            :                                         virt_dev->udev,
    2681                 :            :                                         &virt_dev->eps[i],
    2682                 :            :                                         virt_dev->tt_info);
    2683                 :            :         }
    2684                 :            :         /* Overwrite the information stored in the endpoints' bw_info */
    2685                 :          0 :         xhci_update_bw_info(xhci, virt_dev->in_ctx, ctrl_ctx, virt_dev);
    2686         [ #  # ]:          0 :         for (i = 0; i < 31; i++) {
    2687                 :            :                 /* Add any changed or added endpoints to the interval table */
    2688         [ #  # ]:          0 :                 if (EP_IS_ADDED(ctrl_ctx, i))
    2689                 :          0 :                         xhci_add_ep_to_interval_table(xhci,
    2690                 :            :                                         &virt_dev->eps[i].bw_info,
    2691                 :            :                                         virt_dev->bw_table,
    2692                 :            :                                         virt_dev->udev,
    2693                 :            :                                         &virt_dev->eps[i],
    2694                 :            :                                         virt_dev->tt_info);
    2695                 :            :         }
    2696                 :            : 
    2697         [ #  # ]:          0 :         if (!xhci_check_bw_table(xhci, virt_dev, old_active_eps)) {
    2698                 :            :                 /* Ok, this fits in the bandwidth we have.
    2699                 :            :                  * Update the number of active TTs.
    2700                 :            :                  */
    2701                 :          0 :                 xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
    2702                 :          0 :                 return 0;
    2703                 :            :         }
    2704                 :            : 
    2705                 :            :         /* We don't have enough bandwidth for this, revert the stored info. */
    2706         [ #  # ]:          0 :         for (i = 0; i < 31; i++) {
    2707   [ #  #  #  # ]:          0 :                 if (!EP_IS_ADDED(ctrl_ctx, i) && !EP_IS_DROPPED(ctrl_ctx, i))
    2708                 :          0 :                         continue;
    2709                 :            : 
    2710                 :            :                 /* Drop the new copies of any added or changed endpoints from
    2711                 :            :                  * the interval table.
    2712                 :            :                  */
    2713         [ #  # ]:          0 :                 if (EP_IS_ADDED(ctrl_ctx, i)) {
    2714                 :          0 :                         xhci_drop_ep_from_interval_table(xhci,
    2715                 :            :                                         &virt_dev->eps[i].bw_info,
    2716                 :            :                                         virt_dev->bw_table,
    2717                 :            :                                         virt_dev->udev,
    2718                 :            :                                         &virt_dev->eps[i],
    2719                 :            :                                         virt_dev->tt_info);
    2720                 :            :                 }
    2721                 :            :                 /* Revert the endpoint back to its old information */
    2722                 :          0 :                 memcpy(&virt_dev->eps[i].bw_info, &ep_bw_info[i],
    2723                 :            :                                 sizeof(ep_bw_info[i]));
    2724                 :            :                 /* Add any changed or dropped endpoints back into the table */
    2725         [ #  # ]:          0 :                 if (EP_IS_DROPPED(ctrl_ctx, i))
    2726                 :          0 :                         xhci_add_ep_to_interval_table(xhci,
    2727                 :            :                                         &virt_dev->eps[i].bw_info,
    2728                 :            :                                         virt_dev->bw_table,
    2729                 :            :                                         virt_dev->udev,
    2730                 :            :                                         &virt_dev->eps[i],
    2731                 :            :                                         virt_dev->tt_info);
    2732                 :            :         }
    2733                 :            :         return -ENOMEM;
    2734                 :            : }
    2735                 :            : 
    2736                 :            : 
    2737                 :            : /* Issue a configure endpoint command or evaluate context command
    2738                 :            :  * and wait for it to finish.
    2739                 :            :  */
    2740                 :          0 : static int xhci_configure_endpoint(struct xhci_hcd *xhci,
    2741                 :            :                 struct usb_device *udev,
    2742                 :            :                 struct xhci_command *command,
    2743                 :            :                 bool ctx_change, bool must_succeed)
    2744                 :            : {
    2745                 :          0 :         int ret;
    2746                 :          0 :         unsigned long flags;
    2747                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    2748                 :          0 :         struct xhci_virt_device *virt_dev;
    2749                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    2750                 :            : 
    2751         [ #  # ]:          0 :         if (!command)
    2752                 :            :                 return -EINVAL;
    2753                 :            : 
    2754                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    2755                 :            : 
    2756         [ #  # ]:          0 :         if (xhci->xhc_state & XHCI_STATE_DYING) {
    2757                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    2758                 :          0 :                 return -ESHUTDOWN;
    2759                 :            :         }
    2760                 :            : 
    2761                 :          0 :         virt_dev = xhci->devs[udev->slot_id];
    2762                 :            : 
    2763                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
    2764         [ #  # ]:          0 :         if (!ctrl_ctx) {
    2765                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    2766                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    2767                 :            :                                 __func__);
    2768                 :          0 :                 return -ENOMEM;
    2769                 :            :         }
    2770                 :            : 
    2771   [ #  #  #  # ]:          0 :         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK) &&
    2772                 :          0 :                         xhci_reserve_host_resources(xhci, ctrl_ctx)) {
    2773                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    2774                 :          0 :                 xhci_warn(xhci, "Not enough host resources, "
    2775                 :            :                                 "active endpoint contexts = %u\n",
    2776                 :            :                                 xhci->num_active_eps);
    2777                 :          0 :                 return -ENOMEM;
    2778                 :            :         }
    2779   [ #  #  #  # ]:          0 :         if ((xhci->quirks & XHCI_SW_BW_CHECKING) &&
    2780                 :          0 :             xhci_reserve_bandwidth(xhci, virt_dev, command->in_ctx)) {
    2781         [ #  # ]:          0 :                 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
    2782                 :          0 :                         xhci_free_host_resources(xhci, ctrl_ctx);
    2783                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    2784                 :          0 :                 xhci_warn(xhci, "Not enough bandwidth\n");
    2785                 :          0 :                 return -ENOMEM;
    2786                 :            :         }
    2787                 :            : 
    2788                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
    2789                 :            : 
    2790                 :          0 :         trace_xhci_configure_endpoint_ctrl_ctx(ctrl_ctx);
    2791                 :          0 :         trace_xhci_configure_endpoint(slot_ctx);
    2792                 :            : 
    2793         [ #  # ]:          0 :         if (!ctx_change)
    2794                 :          0 :                 ret = xhci_queue_configure_endpoint(xhci, command,
    2795                 :          0 :                                 command->in_ctx->dma,
    2796                 :          0 :                                 udev->slot_id, must_succeed);
    2797                 :            :         else
    2798                 :          0 :                 ret = xhci_queue_evaluate_context(xhci, command,
    2799                 :          0 :                                 command->in_ctx->dma,
    2800                 :          0 :                                 udev->slot_id, must_succeed);
    2801         [ #  # ]:          0 :         if (ret < 0) {
    2802         [ #  # ]:          0 :                 if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK))
    2803                 :          0 :                         xhci_free_host_resources(xhci, ctrl_ctx);
    2804                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    2805                 :          0 :                 xhci_dbg_trace(xhci,  trace_xhci_dbg_context_change,
    2806                 :            :                                 "FIXME allocate a new ring segment");
    2807                 :          0 :                 return -ENOMEM;
    2808                 :            :         }
    2809                 :          0 :         xhci_ring_cmd_db(xhci);
    2810                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    2811                 :            : 
    2812                 :            :         /* Wait for the configure endpoint command to complete */
    2813                 :          0 :         wait_for_completion(command->completion);
    2814                 :            : 
    2815         [ #  # ]:          0 :         if (!ctx_change)
    2816                 :          0 :                 ret = xhci_configure_endpoint_result(xhci, udev,
    2817                 :            :                                                      &command->status);
    2818                 :            :         else
    2819                 :          0 :                 ret = xhci_evaluate_context_result(xhci, udev,
    2820                 :            :                                                    &command->status);
    2821                 :            : 
    2822         [ #  # ]:          0 :         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
    2823                 :          0 :                 spin_lock_irqsave(&xhci->lock, flags);
    2824                 :            :                 /* If the command failed, remove the reserved resources.
    2825                 :            :                  * Otherwise, clean up the estimate to include dropped eps.
    2826                 :            :                  */
    2827         [ #  # ]:          0 :                 if (ret)
    2828                 :          0 :                         xhci_free_host_resources(xhci, ctrl_ctx);
    2829                 :            :                 else
    2830                 :          0 :                         xhci_finish_resource_reservation(xhci, ctrl_ctx);
    2831                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    2832                 :            :         }
    2833                 :            :         return ret;
    2834                 :            : }
    2835                 :            : 
    2836                 :          0 : static void xhci_check_bw_drop_ep_streams(struct xhci_hcd *xhci,
    2837                 :            :         struct xhci_virt_device *vdev, int i)
    2838                 :            : {
    2839                 :          0 :         struct xhci_virt_ep *ep = &vdev->eps[i];
    2840                 :            : 
    2841         [ #  # ]:          0 :         if (ep->ep_state & EP_HAS_STREAMS) {
    2842         [ #  # ]:          0 :                 xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on set_interface, freeing streams.\n",
    2843                 :            :                                 xhci_get_endpoint_address(i));
    2844                 :          0 :                 xhci_free_stream_info(xhci, ep->stream_info);
    2845                 :          0 :                 ep->stream_info = NULL;
    2846                 :          0 :                 ep->ep_state &= ~EP_HAS_STREAMS;
    2847                 :            :         }
    2848                 :          0 : }
    2849                 :            : 
    2850                 :            : /* Called after one or more calls to xhci_add_endpoint() or
    2851                 :            :  * xhci_drop_endpoint().  If this call fails, the USB core is expected
    2852                 :            :  * to call xhci_reset_bandwidth().
    2853                 :            :  *
    2854                 :            :  * Since we are in the middle of changing either configuration or
    2855                 :            :  * installing a new alt setting, the USB core won't allow URBs to be
    2856                 :            :  * enqueued for any endpoint on the old config or interface.  Nothing
    2857                 :            :  * else should be touching the xhci->devs[slot_id] structure, so we
    2858                 :            :  * don't need to take the xhci->lock for manipulating that.
    2859                 :            :  */
    2860                 :          0 : static int xhci_check_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
    2861                 :            : {
    2862                 :          0 :         int i;
    2863                 :          0 :         int ret = 0;
    2864                 :          0 :         struct xhci_hcd *xhci;
    2865                 :          0 :         struct xhci_virt_device *virt_dev;
    2866                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    2867                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    2868                 :          0 :         struct xhci_command *command;
    2869                 :            : 
    2870                 :          0 :         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
    2871         [ #  # ]:          0 :         if (ret <= 0)
    2872                 :            :                 return ret;
    2873                 :          0 :         xhci = hcd_to_xhci(hcd);
    2874         [ #  # ]:          0 :         if ((xhci->xhc_state & XHCI_STATE_DYING) ||
    2875                 :            :                 (xhci->xhc_state & XHCI_STATE_REMOVING))
    2876                 :            :                 return -ENODEV;
    2877                 :            : 
    2878                 :          0 :         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
    2879                 :          0 :         virt_dev = xhci->devs[udev->slot_id];
    2880                 :            : 
    2881                 :          0 :         command = xhci_alloc_command(xhci, true, GFP_KERNEL);
    2882         [ #  # ]:          0 :         if (!command)
    2883                 :            :                 return -ENOMEM;
    2884                 :            : 
    2885                 :          0 :         command->in_ctx = virt_dev->in_ctx;
    2886                 :            : 
    2887                 :            :         /* See section 4.6.6 - A0 = 1; A1 = D0 = D1 = 0 */
    2888                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
    2889         [ #  # ]:          0 :         if (!ctrl_ctx) {
    2890                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    2891                 :            :                                 __func__);
    2892                 :          0 :                 ret = -ENOMEM;
    2893                 :          0 :                 goto command_cleanup;
    2894                 :            :         }
    2895                 :          0 :         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
    2896                 :          0 :         ctrl_ctx->add_flags &= cpu_to_le32(~EP0_FLAG);
    2897                 :          0 :         ctrl_ctx->drop_flags &= cpu_to_le32(~(SLOT_FLAG | EP0_FLAG));
    2898                 :            : 
    2899                 :            :         /* Don't issue the command if there's no endpoints to update. */
    2900   [ #  #  #  # ]:          0 :         if (ctrl_ctx->add_flags == cpu_to_le32(SLOT_FLAG) &&
    2901                 :            :             ctrl_ctx->drop_flags == 0) {
    2902                 :          0 :                 ret = 0;
    2903                 :          0 :                 goto command_cleanup;
    2904                 :            :         }
    2905                 :            :         /* Fix up Context Entries field. Minimum value is EP0 == BIT(1). */
    2906                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
    2907                 :          0 :         for (i = 31; i >= 1; i--) {
    2908                 :          0 :                 __le32 le32 = cpu_to_le32(BIT(i));
    2909                 :            : 
    2910   [ #  #  #  # ]:          0 :                 if ((virt_dev->eps[i-1].ring && !(ctrl_ctx->drop_flags & le32))
    2911   [ #  #  #  # ]:          0 :                     || (ctrl_ctx->add_flags & le32) || i == 1) {
    2912                 :          0 :                         slot_ctx->dev_info &= cpu_to_le32(~LAST_CTX_MASK);
    2913                 :          0 :                         slot_ctx->dev_info |= cpu_to_le32(LAST_CTX(i));
    2914                 :          0 :                         break;
    2915                 :            :                 }
    2916                 :            :         }
    2917                 :            : 
    2918                 :          0 :         ret = xhci_configure_endpoint(xhci, udev, command,
    2919                 :            :                         false, false);
    2920         [ #  # ]:          0 :         if (ret)
    2921                 :            :                 /* Callee should call reset_bandwidth() */
    2922                 :          0 :                 goto command_cleanup;
    2923                 :            : 
    2924                 :            :         /* Free any rings that were dropped, but not changed. */
    2925         [ #  # ]:          0 :         for (i = 1; i < 31; i++) {
    2926         [ #  # ]:          0 :                 if ((le32_to_cpu(ctrl_ctx->drop_flags) & (1 << (i + 1))) &&
    2927         [ #  # ]:          0 :                     !(le32_to_cpu(ctrl_ctx->add_flags) & (1 << (i + 1)))) {
    2928                 :          0 :                         xhci_free_endpoint_ring(xhci, virt_dev, i);
    2929                 :          0 :                         xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
    2930                 :            :                 }
    2931                 :            :         }
    2932                 :          0 :         xhci_zero_in_ctx(xhci, virt_dev);
    2933                 :            :         /*
    2934                 :            :          * Install any rings for completely new endpoints or changed endpoints,
    2935                 :            :          * and free any old rings from changed endpoints.
    2936                 :            :          */
    2937         [ #  # ]:          0 :         for (i = 1; i < 31; i++) {
    2938         [ #  # ]:          0 :                 if (!virt_dev->eps[i].new_ring)
    2939                 :          0 :                         continue;
    2940                 :            :                 /* Only free the old ring if it exists.
    2941                 :            :                  * It may not if this is the first add of an endpoint.
    2942                 :            :                  */
    2943         [ #  # ]:          0 :                 if (virt_dev->eps[i].ring) {
    2944                 :          0 :                         xhci_free_endpoint_ring(xhci, virt_dev, i);
    2945                 :            :                 }
    2946                 :          0 :                 xhci_check_bw_drop_ep_streams(xhci, virt_dev, i);
    2947                 :          0 :                 virt_dev->eps[i].ring = virt_dev->eps[i].new_ring;
    2948                 :          0 :                 virt_dev->eps[i].new_ring = NULL;
    2949                 :            :         }
    2950                 :          0 : command_cleanup:
    2951                 :          0 :         kfree(command->completion);
    2952                 :          0 :         kfree(command);
    2953                 :            : 
    2954                 :          0 :         return ret;
    2955                 :            : }
    2956                 :            : 
    2957                 :          0 : static void xhci_reset_bandwidth(struct usb_hcd *hcd, struct usb_device *udev)
    2958                 :            : {
    2959                 :          0 :         struct xhci_hcd *xhci;
    2960                 :          0 :         struct xhci_virt_device *virt_dev;
    2961                 :          0 :         int i, ret;
    2962                 :            : 
    2963                 :          0 :         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
    2964         [ #  # ]:          0 :         if (ret <= 0)
    2965                 :            :                 return;
    2966                 :          0 :         xhci = hcd_to_xhci(hcd);
    2967                 :            : 
    2968                 :          0 :         xhci_dbg(xhci, "%s called for udev %p\n", __func__, udev);
    2969                 :          0 :         virt_dev = xhci->devs[udev->slot_id];
    2970                 :            :         /* Free any rings allocated for added endpoints */
    2971         [ #  # ]:          0 :         for (i = 0; i < 31; i++) {
    2972         [ #  # ]:          0 :                 if (virt_dev->eps[i].new_ring) {
    2973                 :          0 :                         xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
    2974                 :          0 :                         xhci_ring_free(xhci, virt_dev->eps[i].new_ring);
    2975                 :          0 :                         virt_dev->eps[i].new_ring = NULL;
    2976                 :            :                 }
    2977                 :            :         }
    2978                 :          0 :         xhci_zero_in_ctx(xhci, virt_dev);
    2979                 :            : }
    2980                 :            : 
    2981                 :          0 : static void xhci_setup_input_ctx_for_config_ep(struct xhci_hcd *xhci,
    2982                 :            :                 struct xhci_container_ctx *in_ctx,
    2983                 :            :                 struct xhci_container_ctx *out_ctx,
    2984                 :            :                 struct xhci_input_control_ctx *ctrl_ctx,
    2985                 :            :                 u32 add_flags, u32 drop_flags)
    2986                 :            : {
    2987                 :          0 :         ctrl_ctx->add_flags = cpu_to_le32(add_flags);
    2988                 :          0 :         ctrl_ctx->drop_flags = cpu_to_le32(drop_flags);
    2989                 :          0 :         xhci_slot_copy(xhci, in_ctx, out_ctx);
    2990                 :          0 :         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
    2991                 :          0 : }
    2992                 :            : 
    2993                 :          0 : static void xhci_setup_input_ctx_for_quirk(struct xhci_hcd *xhci,
    2994                 :            :                 unsigned int slot_id, unsigned int ep_index,
    2995                 :            :                 struct xhci_dequeue_state *deq_state)
    2996                 :            : {
    2997                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    2998                 :          0 :         struct xhci_container_ctx *in_ctx;
    2999                 :          0 :         struct xhci_ep_ctx *ep_ctx;
    3000                 :          0 :         u32 added_ctxs;
    3001                 :          0 :         dma_addr_t addr;
    3002                 :            : 
    3003                 :          0 :         in_ctx = xhci->devs[slot_id]->in_ctx;
    3004                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(in_ctx);
    3005         [ #  # ]:          0 :         if (!ctrl_ctx) {
    3006                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    3007                 :            :                                 __func__);
    3008                 :          0 :                 return;
    3009                 :            :         }
    3010                 :            : 
    3011                 :          0 :         xhci_endpoint_copy(xhci, xhci->devs[slot_id]->in_ctx,
    3012                 :          0 :                         xhci->devs[slot_id]->out_ctx, ep_index);
    3013                 :          0 :         ep_ctx = xhci_get_ep_ctx(xhci, in_ctx, ep_index);
    3014                 :          0 :         addr = xhci_trb_virt_to_dma(deq_state->new_deq_seg,
    3015                 :            :                         deq_state->new_deq_ptr);
    3016         [ #  # ]:          0 :         if (addr == 0) {
    3017                 :          0 :                 xhci_warn(xhci, "WARN Cannot submit config ep after "
    3018                 :            :                                 "reset ep command\n");
    3019                 :          0 :                 xhci_warn(xhci, "WARN deq seg = %p, deq ptr = %p\n",
    3020                 :            :                                 deq_state->new_deq_seg,
    3021                 :            :                                 deq_state->new_deq_ptr);
    3022                 :          0 :                 return;
    3023                 :            :         }
    3024                 :          0 :         ep_ctx->deq = cpu_to_le64(addr | deq_state->new_cycle_state);
    3025                 :            : 
    3026                 :          0 :         added_ctxs = xhci_get_endpoint_flag_from_index(ep_index);
    3027                 :          0 :         xhci_setup_input_ctx_for_config_ep(xhci, xhci->devs[slot_id]->in_ctx,
    3028                 :          0 :                         xhci->devs[slot_id]->out_ctx, ctrl_ctx,
    3029                 :            :                         added_ctxs, added_ctxs);
    3030                 :            : }
    3031                 :            : 
    3032                 :          0 : void xhci_cleanup_stalled_ring(struct xhci_hcd *xhci, unsigned int ep_index,
    3033                 :            :                                unsigned int stream_id, struct xhci_td *td)
    3034                 :            : {
    3035                 :          0 :         struct xhci_dequeue_state deq_state;
    3036                 :          0 :         struct usb_device *udev = td->urb->dev;
    3037                 :            : 
    3038                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
    3039                 :            :                         "Cleaning up stalled endpoint ring");
    3040                 :            :         /* We need to move the HW's dequeue pointer past this TD,
    3041                 :            :          * or it will attempt to resend it on the next doorbell ring.
    3042                 :            :          */
    3043                 :          0 :         xhci_find_new_dequeue_state(xhci, udev->slot_id,
    3044                 :            :                         ep_index, stream_id, td, &deq_state);
    3045                 :            : 
    3046   [ #  #  #  # ]:          0 :         if (!deq_state.new_deq_ptr || !deq_state.new_deq_seg)
    3047                 :          0 :                 return;
    3048                 :            : 
    3049                 :            :         /* HW with the reset endpoint quirk will use the saved dequeue state to
    3050                 :            :          * issue a configure endpoint command later.
    3051                 :            :          */
    3052         [ #  # ]:          0 :         if (!(xhci->quirks & XHCI_RESET_EP_QUIRK)) {
    3053                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_reset_ep,
    3054                 :            :                                 "Queueing new dequeue state");
    3055                 :          0 :                 xhci_queue_new_dequeue_state(xhci, udev->slot_id,
    3056                 :            :                                 ep_index, &deq_state);
    3057                 :            :         } else {
    3058                 :            :                 /* Better hope no one uses the input context between now and the
    3059                 :            :                  * reset endpoint completion!
    3060                 :            :                  * XXX: No idea how this hardware will react when stream rings
    3061                 :            :                  * are enabled.
    3062                 :            :                  */
    3063                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    3064                 :            :                                 "Setting up input context for "
    3065                 :            :                                 "configure endpoint command");
    3066                 :          0 :                 xhci_setup_input_ctx_for_quirk(xhci, udev->slot_id,
    3067                 :            :                                 ep_index, &deq_state);
    3068                 :            :         }
    3069                 :            : }
    3070                 :            : 
    3071                 :          0 : static void xhci_endpoint_disable(struct usb_hcd *hcd,
    3072                 :            :                                   struct usb_host_endpoint *host_ep)
    3073                 :            : {
    3074                 :          0 :         struct xhci_hcd         *xhci;
    3075                 :          0 :         struct xhci_virt_device *vdev;
    3076                 :          0 :         struct xhci_virt_ep     *ep;
    3077                 :          0 :         struct usb_device       *udev;
    3078                 :          0 :         unsigned long           flags;
    3079                 :          0 :         unsigned int            ep_index;
    3080                 :            : 
    3081                 :          0 :         xhci = hcd_to_xhci(hcd);
    3082                 :          0 : rescan:
    3083                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3084                 :            : 
    3085                 :          0 :         udev = (struct usb_device *)host_ep->hcpriv;
    3086   [ #  #  #  # ]:          0 :         if (!udev || !udev->slot_id)
    3087                 :          0 :                 goto done;
    3088                 :            : 
    3089                 :          0 :         vdev = xhci->devs[udev->slot_id];
    3090         [ #  # ]:          0 :         if (!vdev)
    3091                 :          0 :                 goto done;
    3092                 :            : 
    3093         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&host_ep->desc);
    3094                 :          0 :         ep = &vdev->eps[ep_index];
    3095         [ #  # ]:          0 :         if (!ep)
    3096                 :          0 :                 goto done;
    3097                 :            : 
    3098                 :            :         /* wait for hub_tt_work to finish clearing hub TT */
    3099         [ #  # ]:          0 :         if (ep->ep_state & EP_CLEARING_TT) {
    3100                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3101                 :          0 :                 schedule_timeout_uninterruptible(1);
    3102                 :          0 :                 goto rescan;
    3103                 :            :         }
    3104                 :            : 
    3105                 :            :         if (ep->ep_state)
    3106                 :            :                 xhci_dbg(xhci, "endpoint disable with ep_state 0x%x\n",
    3107                 :            :                          ep->ep_state);
    3108                 :          0 : done:
    3109                 :          0 :         host_ep->hcpriv = NULL;
    3110                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3111                 :          0 : }
    3112                 :            : 
    3113                 :            : /*
    3114                 :            :  * Called after usb core issues a clear halt control message.
    3115                 :            :  * The host side of the halt should already be cleared by a reset endpoint
    3116                 :            :  * command issued when the STALL event was received.
    3117                 :            :  *
    3118                 :            :  * The reset endpoint command may only be issued to endpoints in the halted
    3119                 :            :  * state. For software that wishes to reset the data toggle or sequence number
    3120                 :            :  * of an endpoint that isn't in the halted state this function will issue a
    3121                 :            :  * configure endpoint command with the Drop and Add bits set for the target
    3122                 :            :  * endpoint. Refer to the additional note in xhci spcification section 4.6.8.
    3123                 :            :  */
    3124                 :            : 
    3125                 :          0 : static void xhci_endpoint_reset(struct usb_hcd *hcd,
    3126                 :            :                 struct usb_host_endpoint *host_ep)
    3127                 :            : {
    3128                 :          0 :         struct xhci_hcd *xhci;
    3129                 :          0 :         struct usb_device *udev;
    3130                 :          0 :         struct xhci_virt_device *vdev;
    3131                 :          0 :         struct xhci_virt_ep *ep;
    3132                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    3133                 :          0 :         struct xhci_command *stop_cmd, *cfg_cmd;
    3134                 :          0 :         unsigned int ep_index;
    3135                 :          0 :         unsigned long flags;
    3136                 :          0 :         u32 ep_flag;
    3137                 :          0 :         int err;
    3138                 :            : 
    3139                 :          0 :         xhci = hcd_to_xhci(hcd);
    3140         [ #  # ]:          0 :         if (!host_ep->hcpriv)
    3141                 :            :                 return;
    3142                 :          0 :         udev = (struct usb_device *) host_ep->hcpriv;
    3143                 :          0 :         vdev = xhci->devs[udev->slot_id];
    3144                 :            : 
    3145                 :            :         /*
    3146                 :            :          * vdev may be lost due to xHC restore error and re-initialization
    3147                 :            :          * during S3/S4 resume. A new vdev will be allocated later by
    3148                 :            :          * xhci_discover_or_reset_device()
    3149                 :            :          */
    3150   [ #  #  #  # ]:          0 :         if (!udev->slot_id || !vdev)
    3151                 :            :                 return;
    3152         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&host_ep->desc);
    3153                 :          0 :         ep = &vdev->eps[ep_index];
    3154         [ #  # ]:          0 :         if (!ep)
    3155                 :            :                 return;
    3156                 :            : 
    3157                 :            :         /* Bail out if toggle is already being cleared by a endpoint reset */
    3158         [ #  # ]:          0 :         if (ep->ep_state & EP_HARD_CLEAR_TOGGLE) {
    3159                 :          0 :                 ep->ep_state &= ~EP_HARD_CLEAR_TOGGLE;
    3160                 :          0 :                 return;
    3161                 :            :         }
    3162                 :            :         /* Only interrupt and bulk ep's use data toggle, USB2 spec 5.5.4-> */
    3163   [ #  #  #  # ]:          0 :         if (usb_endpoint_xfer_control(&host_ep->desc) ||
    3164                 :            :             usb_endpoint_xfer_isoc(&host_ep->desc))
    3165                 :            :                 return;
    3166                 :            : 
    3167         [ #  # ]:          0 :         ep_flag = xhci_get_endpoint_flag(&host_ep->desc);
    3168                 :            : 
    3169         [ #  # ]:          0 :         if (ep_flag == SLOT_FLAG || ep_flag == EP0_FLAG)
    3170                 :            :                 return;
    3171                 :            : 
    3172                 :          0 :         stop_cmd = xhci_alloc_command(xhci, true, GFP_NOWAIT);
    3173         [ #  # ]:          0 :         if (!stop_cmd)
    3174                 :            :                 return;
    3175                 :            : 
    3176                 :          0 :         cfg_cmd = xhci_alloc_command_with_ctx(xhci, true, GFP_NOWAIT);
    3177         [ #  # ]:          0 :         if (!cfg_cmd)
    3178                 :          0 :                 goto cleanup;
    3179                 :            : 
    3180                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3181                 :            : 
    3182                 :            :         /* block queuing new trbs and ringing ep doorbell */
    3183                 :          0 :         ep->ep_state |= EP_SOFT_CLEAR_TOGGLE;
    3184                 :            : 
    3185                 :            :         /*
    3186                 :            :          * Make sure endpoint ring is empty before resetting the toggle/seq.
    3187                 :            :          * Driver is required to synchronously cancel all transfer request.
    3188                 :            :          * Stop the endpoint to force xHC to update the output context
    3189                 :            :          */
    3190                 :            : 
    3191         [ #  # ]:          0 :         if (!list_empty(&ep->ring->td_list)) {
    3192                 :          0 :                 dev_err(&udev->dev, "EP not empty, refuse reset\n");
    3193                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3194                 :          0 :                 xhci_free_command(xhci, cfg_cmd);
    3195                 :          0 :                 goto cleanup;
    3196                 :            :         }
    3197                 :            : 
    3198                 :          0 :         err = xhci_queue_stop_endpoint(xhci, stop_cmd, udev->slot_id,
    3199                 :            :                                         ep_index, 0);
    3200         [ #  # ]:          0 :         if (err < 0) {
    3201                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3202                 :          0 :                 xhci_free_command(xhci, cfg_cmd);
    3203                 :          0 :                 xhci_dbg(xhci, "%s: Failed to queue stop ep command, %d ",
    3204                 :            :                                 __func__, err);
    3205                 :          0 :                 goto cleanup;
    3206                 :            :         }
    3207                 :            : 
    3208                 :          0 :         xhci_ring_cmd_db(xhci);
    3209                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3210                 :            : 
    3211                 :          0 :         wait_for_completion(stop_cmd->completion);
    3212                 :            : 
    3213                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3214                 :            : 
    3215                 :            :         /* config ep command clears toggle if add and drop ep flags are set */
    3216                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(cfg_cmd->in_ctx);
    3217                 :          0 :         xhci_setup_input_ctx_for_config_ep(xhci, cfg_cmd->in_ctx, vdev->out_ctx,
    3218                 :            :                                            ctrl_ctx, ep_flag, ep_flag);
    3219                 :          0 :         xhci_endpoint_copy(xhci, cfg_cmd->in_ctx, vdev->out_ctx, ep_index);
    3220                 :            : 
    3221                 :          0 :         err = xhci_queue_configure_endpoint(xhci, cfg_cmd, cfg_cmd->in_ctx->dma,
    3222                 :          0 :                                       udev->slot_id, false);
    3223         [ #  # ]:          0 :         if (err < 0) {
    3224                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3225                 :          0 :                 xhci_free_command(xhci, cfg_cmd);
    3226                 :          0 :                 xhci_dbg(xhci, "%s: Failed to queue config ep command, %d ",
    3227                 :            :                                 __func__, err);
    3228                 :          0 :                 goto cleanup;
    3229                 :            :         }
    3230                 :            : 
    3231                 :          0 :         xhci_ring_cmd_db(xhci);
    3232                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3233                 :            : 
    3234                 :          0 :         wait_for_completion(cfg_cmd->completion);
    3235                 :            : 
    3236                 :          0 :         ep->ep_state &= ~EP_SOFT_CLEAR_TOGGLE;
    3237                 :          0 :         xhci_free_command(xhci, cfg_cmd);
    3238                 :          0 : cleanup:
    3239                 :          0 :         xhci_free_command(xhci, stop_cmd);
    3240                 :            : }
    3241                 :            : 
    3242                 :          0 : static int xhci_check_streams_endpoint(struct xhci_hcd *xhci,
    3243                 :            :                 struct usb_device *udev, struct usb_host_endpoint *ep,
    3244                 :            :                 unsigned int slot_id)
    3245                 :            : {
    3246                 :          0 :         int ret;
    3247                 :          0 :         unsigned int ep_index;
    3248                 :          0 :         unsigned int ep_state;
    3249                 :            : 
    3250         [ #  # ]:          0 :         if (!ep)
    3251                 :            :                 return -EINVAL;
    3252                 :          0 :         ret = xhci_check_args(xhci_to_hcd(xhci), udev, ep, 1, true, __func__);
    3253         [ #  # ]:          0 :         if (ret <= 0)
    3254                 :            :                 return -EINVAL;
    3255   [ #  #  #  # ]:          0 :         if (usb_ss_max_streams(&ep->ss_ep_comp) == 0) {
    3256                 :          0 :                 xhci_warn(xhci, "WARN: SuperSpeed Endpoint Companion"
    3257                 :            :                                 " descriptor for ep 0x%x does not support streams\n",
    3258                 :            :                                 ep->desc.bEndpointAddress);
    3259                 :          0 :                 return -EINVAL;
    3260                 :            :         }
    3261                 :            : 
    3262         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&ep->desc);
    3263                 :          0 :         ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
    3264         [ #  # ]:          0 :         if (ep_state & EP_HAS_STREAMS ||
    3265                 :            :                         ep_state & EP_GETTING_STREAMS) {
    3266                 :          0 :                 xhci_warn(xhci, "WARN: SuperSpeed bulk endpoint 0x%x "
    3267                 :            :                                 "already has streams set up.\n",
    3268                 :            :                                 ep->desc.bEndpointAddress);
    3269                 :          0 :                 xhci_warn(xhci, "Send email to xHCI maintainer and ask for "
    3270                 :            :                                 "dynamic stream context array reallocation.\n");
    3271                 :          0 :                 return -EINVAL;
    3272                 :            :         }
    3273         [ #  # ]:          0 :         if (!list_empty(&xhci->devs[slot_id]->eps[ep_index].ring->td_list)) {
    3274                 :          0 :                 xhci_warn(xhci, "Cannot setup streams for SuperSpeed bulk "
    3275                 :            :                                 "endpoint 0x%x; URBs are pending.\n",
    3276                 :            :                                 ep->desc.bEndpointAddress);
    3277                 :          0 :                 return -EINVAL;
    3278                 :            :         }
    3279                 :            :         return 0;
    3280                 :            : }
    3281                 :            : 
    3282                 :            : static void xhci_calculate_streams_entries(struct xhci_hcd *xhci,
    3283                 :            :                 unsigned int *num_streams, unsigned int *num_stream_ctxs)
    3284                 :            : {
    3285                 :            :         unsigned int max_streams;
    3286                 :            : 
    3287                 :            :         /* The stream context array size must be a power of two */
    3288                 :            :         *num_stream_ctxs = roundup_pow_of_two(*num_streams);
    3289                 :            :         /*
    3290                 :            :          * Find out how many primary stream array entries the host controller
    3291                 :            :          * supports.  Later we may use secondary stream arrays (similar to 2nd
    3292                 :            :          * level page entries), but that's an optional feature for xHCI host
    3293                 :            :          * controllers. xHCs must support at least 4 stream IDs.
    3294                 :            :          */
    3295                 :            :         max_streams = HCC_MAX_PSA(xhci->hcc_params);
    3296                 :            :         if (*num_stream_ctxs > max_streams) {
    3297                 :            :                 xhci_dbg(xhci, "xHCI HW only supports %u stream ctx entries.\n",
    3298                 :            :                                 max_streams);
    3299                 :            :                 *num_stream_ctxs = max_streams;
    3300                 :            :                 *num_streams = max_streams;
    3301                 :            :         }
    3302                 :            : }
    3303                 :            : 
    3304                 :            : /* Returns an error code if one of the endpoint already has streams.
    3305                 :            :  * This does not change any data structures, it only checks and gathers
    3306                 :            :  * information.
    3307                 :            :  */
    3308                 :          0 : static int xhci_calculate_streams_and_bitmask(struct xhci_hcd *xhci,
    3309                 :            :                 struct usb_device *udev,
    3310                 :            :                 struct usb_host_endpoint **eps, unsigned int num_eps,
    3311                 :            :                 unsigned int *num_streams, u32 *changed_ep_bitmask)
    3312                 :            : {
    3313                 :          0 :         unsigned int max_streams;
    3314                 :          0 :         unsigned int endpoint_flag;
    3315                 :          0 :         int i;
    3316                 :          0 :         int ret;
    3317                 :            : 
    3318         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3319                 :          0 :                 ret = xhci_check_streams_endpoint(xhci, udev,
    3320                 :          0 :                                 eps[i], udev->slot_id);
    3321         [ #  # ]:          0 :                 if (ret < 0)
    3322                 :          0 :                         return ret;
    3323                 :            : 
    3324         [ #  # ]:          0 :                 max_streams = usb_ss_max_streams(&eps[i]->ss_ep_comp);
    3325         [ #  # ]:          0 :                 if (max_streams < (*num_streams - 1)) {
    3326                 :          0 :                         xhci_dbg(xhci, "Ep 0x%x only supports %u stream IDs.\n",
    3327                 :            :                                         eps[i]->desc.bEndpointAddress,
    3328                 :            :                                         max_streams);
    3329                 :          0 :                         *num_streams = max_streams+1;
    3330                 :            :                 }
    3331                 :            : 
    3332         [ #  # ]:          0 :                 endpoint_flag = xhci_get_endpoint_flag(&eps[i]->desc);
    3333         [ #  # ]:          0 :                 if (*changed_ep_bitmask & endpoint_flag)
    3334                 :            :                         return -EINVAL;
    3335                 :          0 :                 *changed_ep_bitmask |= endpoint_flag;
    3336                 :            :         }
    3337                 :            :         return 0;
    3338                 :            : }
    3339                 :            : 
    3340                 :            : static u32 xhci_calculate_no_streams_bitmask(struct xhci_hcd *xhci,
    3341                 :            :                 struct usb_device *udev,
    3342                 :            :                 struct usb_host_endpoint **eps, unsigned int num_eps)
    3343                 :            : {
    3344                 :            :         u32 changed_ep_bitmask = 0;
    3345                 :            :         unsigned int slot_id;
    3346                 :            :         unsigned int ep_index;
    3347                 :            :         unsigned int ep_state;
    3348                 :            :         int i;
    3349                 :            : 
    3350                 :            :         slot_id = udev->slot_id;
    3351                 :            :         if (!xhci->devs[slot_id])
    3352                 :            :                 return 0;
    3353                 :            : 
    3354                 :            :         for (i = 0; i < num_eps; i++) {
    3355                 :            :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3356                 :            :                 ep_state = xhci->devs[slot_id]->eps[ep_index].ep_state;
    3357                 :            :                 /* Are streams already being freed for the endpoint? */
    3358                 :            :                 if (ep_state & EP_GETTING_NO_STREAMS) {
    3359                 :            :                         xhci_warn(xhci, "WARN Can't disable streams for "
    3360                 :            :                                         "endpoint 0x%x, "
    3361                 :            :                                         "streams are being disabled already\n",
    3362                 :            :                                         eps[i]->desc.bEndpointAddress);
    3363                 :            :                         return 0;
    3364                 :            :                 }
    3365                 :            :                 /* Are there actually any streams to free? */
    3366                 :            :                 if (!(ep_state & EP_HAS_STREAMS) &&
    3367                 :            :                                 !(ep_state & EP_GETTING_STREAMS)) {
    3368                 :            :                         xhci_warn(xhci, "WARN Can't disable streams for "
    3369                 :            :                                         "endpoint 0x%x, "
    3370                 :            :                                         "streams are already disabled!\n",
    3371                 :            :                                         eps[i]->desc.bEndpointAddress);
    3372                 :            :                         xhci_warn(xhci, "WARN xhci_free_streams() called "
    3373                 :            :                                         "with non-streams endpoint\n");
    3374                 :            :                         return 0;
    3375                 :            :                 }
    3376                 :            :                 changed_ep_bitmask |= xhci_get_endpoint_flag(&eps[i]->desc);
    3377                 :            :         }
    3378                 :            :         return changed_ep_bitmask;
    3379                 :            : }
    3380                 :            : 
    3381                 :            : /*
    3382                 :            :  * The USB device drivers use this function (through the HCD interface in USB
    3383                 :            :  * core) to prepare a set of bulk endpoints to use streams.  Streams are used to
    3384                 :            :  * coordinate mass storage command queueing across multiple endpoints (basically
    3385                 :            :  * a stream ID == a task ID).
    3386                 :            :  *
    3387                 :            :  * Setting up streams involves allocating the same size stream context array
    3388                 :            :  * for each endpoint and issuing a configure endpoint command for all endpoints.
    3389                 :            :  *
    3390                 :            :  * Don't allow the call to succeed if one endpoint only supports one stream
    3391                 :            :  * (which means it doesn't support streams at all).
    3392                 :            :  *
    3393                 :            :  * Drivers may get less stream IDs than they asked for, if the host controller
    3394                 :            :  * hardware or endpoints claim they can't support the number of requested
    3395                 :            :  * stream IDs.
    3396                 :            :  */
    3397                 :          0 : static int xhci_alloc_streams(struct usb_hcd *hcd, struct usb_device *udev,
    3398                 :            :                 struct usb_host_endpoint **eps, unsigned int num_eps,
    3399                 :            :                 unsigned int num_streams, gfp_t mem_flags)
    3400                 :            : {
    3401                 :          0 :         int i, ret;
    3402                 :          0 :         struct xhci_hcd *xhci;
    3403                 :          0 :         struct xhci_virt_device *vdev;
    3404                 :          0 :         struct xhci_command *config_cmd;
    3405                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    3406                 :          0 :         unsigned int ep_index;
    3407                 :          0 :         unsigned int num_stream_ctxs;
    3408                 :          0 :         unsigned int max_packet;
    3409                 :          0 :         unsigned long flags;
    3410                 :          0 :         u32 changed_ep_bitmask = 0;
    3411                 :            : 
    3412         [ #  # ]:          0 :         if (!eps)
    3413                 :            :                 return -EINVAL;
    3414                 :            : 
    3415                 :            :         /* Add one to the number of streams requested to account for
    3416                 :            :          * stream 0 that is reserved for xHCI usage.
    3417                 :            :          */
    3418                 :          0 :         num_streams += 1;
    3419                 :          0 :         xhci = hcd_to_xhci(hcd);
    3420                 :          0 :         xhci_dbg(xhci, "Driver wants %u stream IDs (including stream 0).\n",
    3421                 :            :                         num_streams);
    3422                 :            : 
    3423                 :            :         /* MaxPSASize value 0 (2 streams) means streams are not supported */
    3424         [ #  # ]:          0 :         if ((xhci->quirks & XHCI_BROKEN_STREAMS) ||
    3425         [ #  # ]:          0 :                         HCC_MAX_PSA(xhci->hcc_params) < 4) {
    3426                 :            :                 xhci_dbg(xhci, "xHCI controller does not support streams.\n");
    3427                 :            :                 return -ENOSYS;
    3428                 :            :         }
    3429                 :            : 
    3430                 :          0 :         config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
    3431         [ #  # ]:          0 :         if (!config_cmd)
    3432                 :            :                 return -ENOMEM;
    3433                 :            : 
    3434                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
    3435         [ #  # ]:          0 :         if (!ctrl_ctx) {
    3436                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    3437                 :            :                                 __func__);
    3438                 :          0 :                 xhci_free_command(xhci, config_cmd);
    3439                 :          0 :                 return -ENOMEM;
    3440                 :            :         }
    3441                 :            : 
    3442                 :            :         /* Check to make sure all endpoints are not already configured for
    3443                 :            :          * streams.  While we're at it, find the maximum number of streams that
    3444                 :            :          * all the endpoints will support and check for duplicate endpoints.
    3445                 :            :          */
    3446                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3447                 :          0 :         ret = xhci_calculate_streams_and_bitmask(xhci, udev, eps,
    3448                 :            :                         num_eps, &num_streams, &changed_ep_bitmask);
    3449         [ #  # ]:          0 :         if (ret < 0) {
    3450                 :          0 :                 xhci_free_command(xhci, config_cmd);
    3451                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3452                 :          0 :                 return ret;
    3453                 :            :         }
    3454         [ #  # ]:          0 :         if (num_streams <= 1) {
    3455                 :          0 :                 xhci_warn(xhci, "WARN: endpoints can't handle "
    3456                 :            :                                 "more than one stream.\n");
    3457                 :          0 :                 xhci_free_command(xhci, config_cmd);
    3458                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3459                 :          0 :                 return -EINVAL;
    3460                 :            :         }
    3461                 :          0 :         vdev = xhci->devs[udev->slot_id];
    3462                 :            :         /* Mark each endpoint as being in transition, so
    3463                 :            :          * xhci_urb_enqueue() will reject all URBs.
    3464                 :            :          */
    3465         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3466         [ #  # ]:          0 :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3467                 :          0 :                 vdev->eps[ep_index].ep_state |= EP_GETTING_STREAMS;
    3468                 :            :         }
    3469                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3470                 :            : 
    3471                 :            :         /* Setup internal data structures and allocate HW data structures for
    3472                 :            :          * streams (but don't install the HW structures in the input context
    3473                 :            :          * until we're sure all memory allocation succeeded).
    3474                 :            :          */
    3475                 :          0 :         xhci_calculate_streams_entries(xhci, &num_streams, &num_stream_ctxs);
    3476                 :          0 :         xhci_dbg(xhci, "Need %u stream ctx entries for %u stream IDs.\n",
    3477                 :            :                         num_stream_ctxs, num_streams);
    3478                 :            : 
    3479         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3480         [ #  # ]:          0 :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3481                 :          0 :                 max_packet = usb_endpoint_maxp(&eps[i]->desc);
    3482                 :          0 :                 vdev->eps[ep_index].stream_info = xhci_alloc_stream_info(xhci,
    3483                 :            :                                 num_stream_ctxs,
    3484                 :            :                                 num_streams,
    3485                 :            :                                 max_packet, mem_flags);
    3486         [ #  # ]:          0 :                 if (!vdev->eps[ep_index].stream_info)
    3487                 :          0 :                         goto cleanup;
    3488                 :            :                 /* Set maxPstreams in endpoint context and update deq ptr to
    3489                 :            :                  * point to stream context array. FIXME
    3490                 :            :                  */
    3491                 :            :         }
    3492                 :            : 
    3493                 :            :         /* Set up the input context for a configure endpoint command. */
    3494         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3495                 :          0 :                 struct xhci_ep_ctx *ep_ctx;
    3496                 :            : 
    3497         [ #  # ]:          0 :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3498                 :          0 :                 ep_ctx = xhci_get_ep_ctx(xhci, config_cmd->in_ctx, ep_index);
    3499                 :            : 
    3500                 :          0 :                 xhci_endpoint_copy(xhci, config_cmd->in_ctx,
    3501                 :            :                                 vdev->out_ctx, ep_index);
    3502                 :          0 :                 xhci_setup_streams_ep_input_ctx(xhci, ep_ctx,
    3503                 :            :                                 vdev->eps[ep_index].stream_info);
    3504                 :            :         }
    3505                 :            :         /* Tell the HW to drop its old copy of the endpoint context info
    3506                 :            :          * and add the updated copy from the input context.
    3507                 :            :          */
    3508                 :          0 :         xhci_setup_input_ctx_for_config_ep(xhci, config_cmd->in_ctx,
    3509                 :            :                         vdev->out_ctx, ctrl_ctx,
    3510                 :            :                         changed_ep_bitmask, changed_ep_bitmask);
    3511                 :            : 
    3512                 :            :         /* Issue and wait for the configure endpoint command */
    3513                 :          0 :         ret = xhci_configure_endpoint(xhci, udev, config_cmd,
    3514                 :            :                         false, false);
    3515                 :            : 
    3516                 :            :         /* xHC rejected the configure endpoint command for some reason, so we
    3517                 :            :          * leave the old ring intact and free our internal streams data
    3518                 :            :          * structure.
    3519                 :            :          */
    3520         [ #  # ]:          0 :         if (ret < 0)
    3521                 :          0 :                 goto cleanup;
    3522                 :            : 
    3523                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3524         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3525         [ #  # ]:          0 :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3526                 :          0 :                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
    3527                 :          0 :                 xhci_dbg(xhci, "Slot %u ep ctx %u now has streams.\n",
    3528                 :            :                          udev->slot_id, ep_index);
    3529                 :          0 :                 vdev->eps[ep_index].ep_state |= EP_HAS_STREAMS;
    3530                 :            :         }
    3531                 :          0 :         xhci_free_command(xhci, config_cmd);
    3532                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3533                 :            : 
    3534                 :            :         /* Subtract 1 for stream 0, which drivers can't use */
    3535                 :          0 :         return num_streams - 1;
    3536                 :            : 
    3537                 :            : cleanup:
    3538                 :            :         /* If it didn't work, free the streams! */
    3539         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3540         [ #  # ]:          0 :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3541                 :          0 :                 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
    3542                 :          0 :                 vdev->eps[ep_index].stream_info = NULL;
    3543                 :            :                 /* FIXME Unset maxPstreams in endpoint context and
    3544                 :            :                  * update deq ptr to point to normal string ring.
    3545                 :            :                  */
    3546                 :          0 :                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_STREAMS;
    3547                 :          0 :                 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
    3548                 :          0 :                 xhci_endpoint_zero(xhci, vdev, eps[i]);
    3549                 :            :         }
    3550                 :          0 :         xhci_free_command(xhci, config_cmd);
    3551                 :          0 :         return -ENOMEM;
    3552                 :            : }
    3553                 :            : 
    3554                 :            : /* Transition the endpoint from using streams to being a "normal" endpoint
    3555                 :            :  * without streams.
    3556                 :            :  *
    3557                 :            :  * Modify the endpoint context state, submit a configure endpoint command,
    3558                 :            :  * and free all endpoint rings for streams if that completes successfully.
    3559                 :            :  */
    3560                 :          0 : static int xhci_free_streams(struct usb_hcd *hcd, struct usb_device *udev,
    3561                 :            :                 struct usb_host_endpoint **eps, unsigned int num_eps,
    3562                 :            :                 gfp_t mem_flags)
    3563                 :            : {
    3564                 :          0 :         int i, ret;
    3565                 :          0 :         struct xhci_hcd *xhci;
    3566                 :          0 :         struct xhci_virt_device *vdev;
    3567                 :          0 :         struct xhci_command *command;
    3568                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    3569                 :          0 :         unsigned int ep_index;
    3570                 :          0 :         unsigned long flags;
    3571                 :          0 :         u32 changed_ep_bitmask;
    3572                 :            : 
    3573                 :          0 :         xhci = hcd_to_xhci(hcd);
    3574                 :          0 :         vdev = xhci->devs[udev->slot_id];
    3575                 :            : 
    3576                 :            :         /* Set up a configure endpoint command to remove the streams rings */
    3577                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3578                 :          0 :         changed_ep_bitmask = xhci_calculate_no_streams_bitmask(xhci,
    3579                 :            :                         udev, eps, num_eps);
    3580         [ #  # ]:          0 :         if (changed_ep_bitmask == 0) {
    3581                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3582                 :          0 :                 return -EINVAL;
    3583                 :            :         }
    3584                 :            : 
    3585                 :            :         /* Use the xhci_command structure from the first endpoint.  We may have
    3586                 :            :          * allocated too many, but the driver may call xhci_free_streams() for
    3587                 :            :          * each endpoint it grouped into one call to xhci_alloc_streams().
    3588                 :            :          */
    3589         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&eps[0]->desc);
    3590                 :          0 :         command = vdev->eps[ep_index].stream_info->free_streams_command;
    3591                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
    3592         [ #  # ]:          0 :         if (!ctrl_ctx) {
    3593                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3594                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    3595                 :            :                                 __func__);
    3596                 :          0 :                 return -EINVAL;
    3597                 :            :         }
    3598                 :            : 
    3599         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3600                 :          0 :                 struct xhci_ep_ctx *ep_ctx;
    3601                 :            : 
    3602         [ #  # ]:          0 :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3603                 :          0 :                 ep_ctx = xhci_get_ep_ctx(xhci, command->in_ctx, ep_index);
    3604                 :          0 :                 xhci->devs[udev->slot_id]->eps[ep_index].ep_state |=
    3605                 :            :                         EP_GETTING_NO_STREAMS;
    3606                 :            : 
    3607                 :          0 :                 xhci_endpoint_copy(xhci, command->in_ctx,
    3608                 :            :                                 vdev->out_ctx, ep_index);
    3609                 :          0 :                 xhci_setup_no_streams_ep_input_ctx(ep_ctx,
    3610                 :            :                                 &vdev->eps[ep_index]);
    3611                 :            :         }
    3612                 :          0 :         xhci_setup_input_ctx_for_config_ep(xhci, command->in_ctx,
    3613                 :            :                         vdev->out_ctx, ctrl_ctx,
    3614                 :            :                         changed_ep_bitmask, changed_ep_bitmask);
    3615                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3616                 :            : 
    3617                 :            :         /* Issue and wait for the configure endpoint command,
    3618                 :            :          * which must succeed.
    3619                 :            :          */
    3620                 :          0 :         ret = xhci_configure_endpoint(xhci, udev, command,
    3621                 :            :                         false, true);
    3622                 :            : 
    3623                 :            :         /* xHC rejected the configure endpoint command for some reason, so we
    3624                 :            :          * leave the streams rings intact.
    3625                 :            :          */
    3626         [ #  # ]:          0 :         if (ret < 0)
    3627                 :            :                 return ret;
    3628                 :            : 
    3629                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3630         [ #  # ]:          0 :         for (i = 0; i < num_eps; i++) {
    3631         [ #  # ]:          0 :                 ep_index = xhci_get_endpoint_index(&eps[i]->desc);
    3632                 :          0 :                 xhci_free_stream_info(xhci, vdev->eps[ep_index].stream_info);
    3633                 :          0 :                 vdev->eps[ep_index].stream_info = NULL;
    3634                 :            :                 /* FIXME Unset maxPstreams in endpoint context and
    3635                 :            :                  * update deq ptr to point to normal string ring.
    3636                 :            :                  */
    3637                 :          0 :                 vdev->eps[ep_index].ep_state &= ~EP_GETTING_NO_STREAMS;
    3638                 :          0 :                 vdev->eps[ep_index].ep_state &= ~EP_HAS_STREAMS;
    3639                 :            :         }
    3640                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3641                 :            : 
    3642                 :          0 :         return 0;
    3643                 :            : }
    3644                 :            : 
    3645                 :            : /*
    3646                 :            :  * Deletes endpoint resources for endpoints that were active before a Reset
    3647                 :            :  * Device command, or a Disable Slot command.  The Reset Device command leaves
    3648                 :            :  * the control endpoint intact, whereas the Disable Slot command deletes it.
    3649                 :            :  *
    3650                 :            :  * Must be called with xhci->lock held.
    3651                 :            :  */
    3652                 :          0 : void xhci_free_device_endpoint_resources(struct xhci_hcd *xhci,
    3653                 :            :         struct xhci_virt_device *virt_dev, bool drop_control_ep)
    3654                 :            : {
    3655                 :          0 :         int i;
    3656                 :          0 :         unsigned int num_dropped_eps = 0;
    3657                 :          0 :         unsigned int drop_flags = 0;
    3658                 :            : 
    3659         [ #  # ]:          0 :         for (i = (drop_control_ep ? 0 : 1); i < 31; i++) {
    3660         [ #  # ]:          0 :                 if (virt_dev->eps[i].ring) {
    3661                 :          0 :                         drop_flags |= 1 << i;
    3662                 :          0 :                         num_dropped_eps++;
    3663                 :            :                 }
    3664                 :            :         }
    3665                 :          0 :         xhci->num_active_eps -= num_dropped_eps;
    3666         [ #  # ]:          0 :         if (num_dropped_eps)
    3667                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    3668                 :            :                                 "Dropped %u ep ctxs, flags = 0x%x, "
    3669                 :            :                                 "%u now active.",
    3670                 :            :                                 num_dropped_eps, drop_flags,
    3671                 :            :                                 xhci->num_active_eps);
    3672                 :          0 : }
    3673                 :            : 
    3674                 :            : /*
    3675                 :            :  * This submits a Reset Device Command, which will set the device state to 0,
    3676                 :            :  * set the device address to 0, and disable all the endpoints except the default
    3677                 :            :  * control endpoint.  The USB core should come back and call
    3678                 :            :  * xhci_address_device(), and then re-set up the configuration.  If this is
    3679                 :            :  * called because of a usb_reset_and_verify_device(), then the old alternate
    3680                 :            :  * settings will be re-installed through the normal bandwidth allocation
    3681                 :            :  * functions.
    3682                 :            :  *
    3683                 :            :  * Wait for the Reset Device command to finish.  Remove all structures
    3684                 :            :  * associated with the endpoints that were disabled.  Clear the input device
    3685                 :            :  * structure? Reset the control endpoint 0 max packet size?
    3686                 :            :  *
    3687                 :            :  * If the virt_dev to be reset does not exist or does not match the udev,
    3688                 :            :  * it means the device is lost, possibly due to the xHC restore error and
    3689                 :            :  * re-initialization during S3/S4. In this case, call xhci_alloc_dev() to
    3690                 :            :  * re-allocate the device.
    3691                 :            :  */
    3692                 :          0 : static int xhci_discover_or_reset_device(struct usb_hcd *hcd,
    3693                 :            :                 struct usb_device *udev)
    3694                 :            : {
    3695                 :          0 :         int ret, i;
    3696                 :          0 :         unsigned long flags;
    3697                 :          0 :         struct xhci_hcd *xhci;
    3698                 :          0 :         unsigned int slot_id;
    3699                 :          0 :         struct xhci_virt_device *virt_dev;
    3700                 :          0 :         struct xhci_command *reset_device_cmd;
    3701                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    3702                 :          0 :         int old_active_eps = 0;
    3703                 :            : 
    3704                 :          0 :         ret = xhci_check_args(hcd, udev, NULL, 0, false, __func__);
    3705         [ #  # ]:          0 :         if (ret <= 0)
    3706                 :            :                 return ret;
    3707                 :          0 :         xhci = hcd_to_xhci(hcd);
    3708                 :          0 :         slot_id = udev->slot_id;
    3709                 :          0 :         virt_dev = xhci->devs[slot_id];
    3710         [ #  # ]:          0 :         if (!virt_dev) {
    3711                 :          0 :                 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
    3712                 :            :                                 "not exist. Re-allocate the device\n", slot_id);
    3713                 :          0 :                 ret = xhci_alloc_dev(hcd, udev);
    3714         [ #  # ]:          0 :                 if (ret == 1)
    3715                 :            :                         return 0;
    3716                 :            :                 else
    3717                 :          0 :                         return -EINVAL;
    3718                 :            :         }
    3719                 :            : 
    3720         [ #  # ]:          0 :         if (virt_dev->tt_info)
    3721                 :          0 :                 old_active_eps = virt_dev->tt_info->active_eps;
    3722                 :            : 
    3723         [ #  # ]:          0 :         if (virt_dev->udev != udev) {
    3724                 :            :                 /* If the virt_dev and the udev does not match, this virt_dev
    3725                 :            :                  * may belong to another udev.
    3726                 :            :                  * Re-allocate the device.
    3727                 :            :                  */
    3728                 :          0 :                 xhci_dbg(xhci, "The device to be reset with slot ID %u does "
    3729                 :            :                                 "not match the udev. Re-allocate the device\n",
    3730                 :            :                                 slot_id);
    3731                 :          0 :                 ret = xhci_alloc_dev(hcd, udev);
    3732         [ #  # ]:          0 :                 if (ret == 1)
    3733                 :            :                         return 0;
    3734                 :            :                 else
    3735                 :          0 :                         return -EINVAL;
    3736                 :            :         }
    3737                 :            : 
    3738                 :            :         /* If device is not setup, there is no point in resetting it */
    3739                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
    3740         [ #  # ]:          0 :         if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
    3741                 :            :                                                 SLOT_STATE_DISABLED)
    3742                 :            :                 return 0;
    3743                 :            : 
    3744                 :          0 :         trace_xhci_discover_or_reset_device(slot_ctx);
    3745                 :            : 
    3746                 :          0 :         xhci_dbg(xhci, "Resetting device with slot ID %u\n", slot_id);
    3747                 :            :         /* Allocate the command structure that holds the struct completion.
    3748                 :            :          * Assume we're in process context, since the normal device reset
    3749                 :            :          * process has to wait for the device anyway.  Storage devices are
    3750                 :            :          * reset as part of error handling, so use GFP_NOIO instead of
    3751                 :            :          * GFP_KERNEL.
    3752                 :            :          */
    3753                 :          0 :         reset_device_cmd = xhci_alloc_command(xhci, true, GFP_NOIO);
    3754         [ #  # ]:          0 :         if (!reset_device_cmd) {
    3755                 :            :                 xhci_dbg(xhci, "Couldn't allocate command structure.\n");
    3756                 :            :                 return -ENOMEM;
    3757                 :            :         }
    3758                 :            : 
    3759                 :            :         /* Attempt to submit the Reset Device command to the command ring */
    3760                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3761                 :            : 
    3762                 :          0 :         ret = xhci_queue_reset_device(xhci, reset_device_cmd, slot_id);
    3763         [ #  # ]:          0 :         if (ret) {
    3764                 :          0 :                 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
    3765                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3766                 :          0 :                 goto command_cleanup;
    3767                 :            :         }
    3768                 :          0 :         xhci_ring_cmd_db(xhci);
    3769                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3770                 :            : 
    3771                 :            :         /* Wait for the Reset Device command to finish */
    3772                 :          0 :         wait_for_completion(reset_device_cmd->completion);
    3773                 :            : 
    3774                 :            :         /* The Reset Device command can't fail, according to the 0.95/0.96 spec,
    3775                 :            :          * unless we tried to reset a slot ID that wasn't enabled,
    3776                 :            :          * or the device wasn't in the addressed or configured state.
    3777                 :            :          */
    3778                 :          0 :         ret = reset_device_cmd->status;
    3779   [ #  #  #  # ]:          0 :         switch (ret) {
    3780                 :          0 :         case COMP_COMMAND_ABORTED:
    3781                 :            :         case COMP_COMMAND_RING_STOPPED:
    3782                 :          0 :                 xhci_warn(xhci, "Timeout waiting for reset device command\n");
    3783                 :          0 :                 ret = -ETIME;
    3784                 :          0 :                 goto command_cleanup;
    3785                 :            :         case COMP_SLOT_NOT_ENABLED_ERROR: /* 0.95 completion for bad slot ID */
    3786                 :            :         case COMP_CONTEXT_STATE_ERROR: /* 0.96 completion code for same thing */
    3787                 :          0 :                 xhci_dbg(xhci, "Can't reset device (slot ID %u) in %s state\n",
    3788                 :            :                                 slot_id,
    3789                 :            :                                 xhci_get_slot_state(xhci, virt_dev->out_ctx));
    3790                 :          0 :                 xhci_dbg(xhci, "Not freeing device rings.\n");
    3791                 :            :                 /* Don't treat this as an error.  May change my mind later. */
    3792                 :          0 :                 ret = 0;
    3793                 :          0 :                 goto command_cleanup;
    3794                 :            :         case COMP_SUCCESS:
    3795                 :            :                 xhci_dbg(xhci, "Successful reset device command.\n");
    3796                 :            :                 break;
    3797                 :          0 :         default:
    3798         [ #  # ]:          0 :                 if (xhci_is_vendor_info_code(xhci, ret))
    3799                 :            :                         break;
    3800                 :          0 :                 xhci_warn(xhci, "Unknown completion code %u for "
    3801                 :            :                                 "reset device command.\n", ret);
    3802                 :          0 :                 ret = -EINVAL;
    3803                 :          0 :                 goto command_cleanup;
    3804                 :            :         }
    3805                 :            : 
    3806                 :            :         /* Free up host controller endpoint resources */
    3807         [ #  # ]:          0 :         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
    3808                 :          0 :                 spin_lock_irqsave(&xhci->lock, flags);
    3809                 :            :                 /* Don't delete the default control endpoint resources */
    3810                 :          0 :                 xhci_free_device_endpoint_resources(xhci, virt_dev, false);
    3811                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3812                 :            :         }
    3813                 :            : 
    3814                 :            :         /* Everything but endpoint 0 is disabled, so free the rings. */
    3815         [ #  # ]:          0 :         for (i = 1; i < 31; i++) {
    3816                 :          0 :                 struct xhci_virt_ep *ep = &virt_dev->eps[i];
    3817                 :            : 
    3818         [ #  # ]:          0 :                 if (ep->ep_state & EP_HAS_STREAMS) {
    3819         [ #  # ]:          0 :                         xhci_warn(xhci, "WARN: endpoint 0x%02x has streams on device reset, freeing streams.\n",
    3820                 :            :                                         xhci_get_endpoint_address(i));
    3821                 :          0 :                         xhci_free_stream_info(xhci, ep->stream_info);
    3822                 :          0 :                         ep->stream_info = NULL;
    3823                 :          0 :                         ep->ep_state &= ~EP_HAS_STREAMS;
    3824                 :            :                 }
    3825                 :            : 
    3826         [ #  # ]:          0 :                 if (ep->ring) {
    3827                 :          0 :                         xhci_debugfs_remove_endpoint(xhci, virt_dev, i);
    3828                 :          0 :                         xhci_free_endpoint_ring(xhci, virt_dev, i);
    3829                 :            :                 }
    3830         [ #  # ]:          0 :                 if (!list_empty(&virt_dev->eps[i].bw_endpoint_list))
    3831                 :          0 :                         xhci_drop_ep_from_interval_table(xhci,
    3832                 :            :                                         &virt_dev->eps[i].bw_info,
    3833                 :            :                                         virt_dev->bw_table,
    3834                 :            :                                         udev,
    3835                 :            :                                         &virt_dev->eps[i],
    3836                 :            :                                         virt_dev->tt_info);
    3837                 :          0 :                 xhci_clear_endpoint_bw_info(&virt_dev->eps[i].bw_info);
    3838                 :            :         }
    3839                 :            :         /* If necessary, update the number of active TTs on this root port */
    3840                 :          0 :         xhci_update_tt_active_eps(xhci, virt_dev, old_active_eps);
    3841                 :          0 :         virt_dev->flags = 0;
    3842                 :          0 :         ret = 0;
    3843                 :            : 
    3844                 :          0 : command_cleanup:
    3845                 :          0 :         xhci_free_command(xhci, reset_device_cmd);
    3846                 :          0 :         return ret;
    3847                 :            : }
    3848                 :            : 
    3849                 :            : /*
    3850                 :            :  * At this point, the struct usb_device is about to go away, the device has
    3851                 :            :  * disconnected, and all traffic has been stopped and the endpoints have been
    3852                 :            :  * disabled.  Free any HC data structures associated with that device.
    3853                 :            :  */
    3854                 :          0 : static void xhci_free_dev(struct usb_hcd *hcd, struct usb_device *udev)
    3855                 :            : {
    3856                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    3857                 :          0 :         struct xhci_virt_device *virt_dev;
    3858                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    3859                 :          0 :         int i, ret;
    3860                 :            : 
    3861                 :            : #ifndef CONFIG_USB_DEFAULT_PERSIST
    3862                 :            :         /*
    3863                 :            :          * We called pm_runtime_get_noresume when the device was attached.
    3864                 :            :          * Decrement the counter here to allow controller to runtime suspend
    3865                 :            :          * if no devices remain.
    3866                 :            :          */
    3867                 :            :         if (xhci->quirks & XHCI_RESET_ON_RESUME)
    3868                 :            :                 pm_runtime_put_noidle(hcd->self.controller);
    3869                 :            : #endif
    3870                 :            : 
    3871                 :          0 :         ret = xhci_check_args(hcd, udev, NULL, 0, true, __func__);
    3872                 :            :         /* If the host is halted due to driver unload, we still need to free the
    3873                 :            :          * device.
    3874                 :            :          */
    3875         [ #  # ]:          0 :         if (ret <= 0 && ret != -ENODEV)
    3876                 :            :                 return;
    3877                 :            : 
    3878                 :          0 :         virt_dev = xhci->devs[udev->slot_id];
    3879                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
    3880                 :          0 :         trace_xhci_free_dev(slot_ctx);
    3881                 :            : 
    3882                 :            :         /* Stop any wayward timer functions (which may grab the lock) */
    3883         [ #  # ]:          0 :         for (i = 0; i < 31; i++) {
    3884                 :          0 :                 virt_dev->eps[i].ep_state &= ~EP_STOP_CMD_PENDING;
    3885                 :          0 :                 del_timer_sync(&virt_dev->eps[i].stop_cmd_timer);
    3886                 :            :         }
    3887                 :          0 :         virt_dev->udev = NULL;
    3888                 :          0 :         ret = xhci_disable_slot(xhci, udev->slot_id);
    3889         [ #  # ]:          0 :         if (ret)
    3890                 :          0 :                 xhci_free_virt_device(xhci, udev->slot_id);
    3891                 :            : }
    3892                 :            : 
    3893                 :          0 : int xhci_disable_slot(struct xhci_hcd *xhci, u32 slot_id)
    3894                 :            : {
    3895                 :          0 :         struct xhci_command *command;
    3896                 :          0 :         unsigned long flags;
    3897                 :          0 :         u32 state;
    3898                 :          0 :         int ret = 0;
    3899                 :            : 
    3900                 :          0 :         command = xhci_alloc_command(xhci, false, GFP_KERNEL);
    3901         [ #  # ]:          0 :         if (!command)
    3902                 :            :                 return -ENOMEM;
    3903                 :            : 
    3904                 :          0 :         xhci_debugfs_remove_slot(xhci, slot_id);
    3905                 :            : 
    3906                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3907                 :            :         /* Don't disable the slot if the host controller is dead. */
    3908                 :          0 :         state = readl(&xhci->op_regs->status);
    3909   [ #  #  #  # ]:          0 :         if (state == 0xffffffff || (xhci->xhc_state & XHCI_STATE_DYING) ||
    3910                 :            :                         (xhci->xhc_state & XHCI_STATE_HALTED)) {
    3911                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3912                 :          0 :                 kfree(command);
    3913                 :          0 :                 return -ENODEV;
    3914                 :            :         }
    3915                 :            : 
    3916                 :          0 :         ret = xhci_queue_slot_control(xhci, command, TRB_DISABLE_SLOT,
    3917                 :            :                                 slot_id);
    3918         [ #  # ]:          0 :         if (ret) {
    3919                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3920                 :          0 :                 kfree(command);
    3921                 :          0 :                 return ret;
    3922                 :            :         }
    3923                 :          0 :         xhci_ring_cmd_db(xhci);
    3924                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3925                 :          0 :         return ret;
    3926                 :            : }
    3927                 :            : 
    3928                 :            : /*
    3929                 :            :  * Checks if we have enough host controller resources for the default control
    3930                 :            :  * endpoint.
    3931                 :            :  *
    3932                 :            :  * Must be called with xhci->lock held.
    3933                 :            :  */
    3934                 :          0 : static int xhci_reserve_host_control_ep_resources(struct xhci_hcd *xhci)
    3935                 :            : {
    3936         [ #  # ]:          0 :         if (xhci->num_active_eps + 1 > xhci->limit_active_eps) {
    3937                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    3938                 :            :                                 "Not enough ep ctxs: "
    3939                 :            :                                 "%u active, need to add 1, limit is %u.",
    3940                 :            :                                 xhci->num_active_eps, xhci->limit_active_eps);
    3941                 :          0 :                 return -ENOMEM;
    3942                 :            :         }
    3943                 :          0 :         xhci->num_active_eps += 1;
    3944                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_quirks,
    3945                 :            :                         "Adding 1 ep ctx, %u now active.",
    3946                 :            :                         xhci->num_active_eps);
    3947                 :          0 :         return 0;
    3948                 :            : }
    3949                 :            : 
    3950                 :            : 
    3951                 :            : /*
    3952                 :            :  * Returns 0 if the xHC ran out of device slots, the Enable Slot command
    3953                 :            :  * timed out, or allocating memory failed.  Returns 1 on success.
    3954                 :            :  */
    3955                 :          0 : int xhci_alloc_dev(struct usb_hcd *hcd, struct usb_device *udev)
    3956                 :            : {
    3957                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    3958                 :          0 :         struct xhci_virt_device *vdev;
    3959                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    3960                 :          0 :         unsigned long flags;
    3961                 :          0 :         int ret, slot_id;
    3962                 :          0 :         struct xhci_command *command;
    3963                 :            : 
    3964                 :          0 :         command = xhci_alloc_command(xhci, true, GFP_KERNEL);
    3965         [ #  # ]:          0 :         if (!command)
    3966                 :            :                 return 0;
    3967                 :            : 
    3968                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    3969                 :          0 :         ret = xhci_queue_slot_control(xhci, command, TRB_ENABLE_SLOT, 0);
    3970         [ #  # ]:          0 :         if (ret) {
    3971                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    3972                 :          0 :                 xhci_dbg(xhci, "FIXME: allocate a command ring segment\n");
    3973                 :          0 :                 xhci_free_command(xhci, command);
    3974                 :          0 :                 return 0;
    3975                 :            :         }
    3976                 :          0 :         xhci_ring_cmd_db(xhci);
    3977                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    3978                 :            : 
    3979                 :          0 :         wait_for_completion(command->completion);
    3980                 :          0 :         slot_id = command->slot_id;
    3981                 :            : 
    3982   [ #  #  #  # ]:          0 :         if (!slot_id || command->status != COMP_SUCCESS) {
    3983                 :          0 :                 xhci_err(xhci, "Error while assigning device slot ID\n");
    3984                 :          0 :                 xhci_err(xhci, "Max number of devices this xHCI host supports is %u.\n",
    3985                 :            :                                 HCS_MAX_SLOTS(
    3986                 :            :                                         readl(&xhci->cap_regs->hcs_params1)));
    3987                 :          0 :                 xhci_free_command(xhci, command);
    3988                 :          0 :                 return 0;
    3989                 :            :         }
    3990                 :            : 
    3991                 :          0 :         xhci_free_command(xhci, command);
    3992                 :            : 
    3993         [ #  # ]:          0 :         if ((xhci->quirks & XHCI_EP_LIMIT_QUIRK)) {
    3994                 :          0 :                 spin_lock_irqsave(&xhci->lock, flags);
    3995                 :          0 :                 ret = xhci_reserve_host_control_ep_resources(xhci);
    3996         [ #  # ]:          0 :                 if (ret) {
    3997                 :          0 :                         spin_unlock_irqrestore(&xhci->lock, flags);
    3998                 :          0 :                         xhci_warn(xhci, "Not enough host resources, "
    3999                 :            :                                         "active endpoint contexts = %u\n",
    4000                 :            :                                         xhci->num_active_eps);
    4001                 :          0 :                         goto disable_slot;
    4002                 :            :                 }
    4003                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    4004                 :            :         }
    4005                 :            :         /* Use GFP_NOIO, since this function can be called from
    4006                 :            :          * xhci_discover_or_reset_device(), which may be called as part of
    4007                 :            :          * mass storage driver error handling.
    4008                 :            :          */
    4009         [ #  # ]:          0 :         if (!xhci_alloc_virt_device(xhci, slot_id, udev, GFP_NOIO)) {
    4010                 :          0 :                 xhci_warn(xhci, "Could not allocate xHCI USB device data structures\n");
    4011                 :          0 :                 goto disable_slot;
    4012                 :            :         }
    4013                 :          0 :         vdev = xhci->devs[slot_id];
    4014                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, vdev->out_ctx);
    4015                 :          0 :         trace_xhci_alloc_dev(slot_ctx);
    4016                 :            : 
    4017                 :          0 :         udev->slot_id = slot_id;
    4018                 :            : 
    4019                 :          0 :         xhci_debugfs_create_slot(xhci, slot_id);
    4020                 :            : 
    4021                 :            : #ifndef CONFIG_USB_DEFAULT_PERSIST
    4022                 :            :         /*
    4023                 :            :          * If resetting upon resume, we can't put the controller into runtime
    4024                 :            :          * suspend if there is a device attached.
    4025                 :            :          */
    4026                 :            :         if (xhci->quirks & XHCI_RESET_ON_RESUME)
    4027                 :            :                 pm_runtime_get_noresume(hcd->self.controller);
    4028                 :            : #endif
    4029                 :            : 
    4030                 :            :         /* Is this a LS or FS device under a HS hub? */
    4031                 :            :         /* Hub or peripherial? */
    4032                 :          0 :         return 1;
    4033                 :            : 
    4034                 :          0 : disable_slot:
    4035                 :          0 :         ret = xhci_disable_slot(xhci, udev->slot_id);
    4036         [ #  # ]:          0 :         if (ret)
    4037                 :          0 :                 xhci_free_virt_device(xhci, udev->slot_id);
    4038                 :            : 
    4039                 :            :         return 0;
    4040                 :            : }
    4041                 :            : 
    4042                 :            : /*
    4043                 :            :  * Issue an Address Device command and optionally send a corresponding
    4044                 :            :  * SetAddress request to the device.
    4045                 :            :  */
    4046                 :          0 : static int xhci_setup_device(struct usb_hcd *hcd, struct usb_device *udev,
    4047                 :            :                              enum xhci_setup_dev setup)
    4048                 :            : {
    4049         [ #  # ]:          0 :         const char *act = setup == SETUP_CONTEXT_ONLY ? "context" : "address";
    4050                 :          0 :         unsigned long flags;
    4051                 :          0 :         struct xhci_virt_device *virt_dev;
    4052                 :          0 :         int ret = 0;
    4053                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    4054                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    4055                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    4056                 :          0 :         u64 temp_64;
    4057                 :          0 :         struct xhci_command *command = NULL;
    4058                 :            : 
    4059                 :          0 :         mutex_lock(&xhci->mutex);
    4060                 :            : 
    4061         [ #  # ]:          0 :         if (xhci->xhc_state) {       /* dying, removing or halted */
    4062                 :          0 :                 ret = -ESHUTDOWN;
    4063                 :          0 :                 goto out;
    4064                 :            :         }
    4065                 :            : 
    4066         [ #  # ]:          0 :         if (!udev->slot_id) {
    4067                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
    4068                 :            :                                 "Bad Slot ID %d", udev->slot_id);
    4069                 :          0 :                 ret = -EINVAL;
    4070                 :          0 :                 goto out;
    4071                 :            :         }
    4072                 :            : 
    4073                 :          0 :         virt_dev = xhci->devs[udev->slot_id];
    4074                 :            : 
    4075   [ #  #  #  # ]:          0 :         if (WARN_ON(!virt_dev)) {
    4076                 :            :                 /*
    4077                 :            :                  * In plug/unplug torture test with an NEC controller,
    4078                 :            :                  * a zero-dereference was observed once due to virt_dev = 0.
    4079                 :            :                  * Print useful debug rather than crash if it is observed again!
    4080                 :            :                  */
    4081                 :          0 :                 xhci_warn(xhci, "Virt dev invalid for slot_id 0x%x!\n",
    4082                 :            :                         udev->slot_id);
    4083                 :          0 :                 ret = -EINVAL;
    4084                 :          0 :                 goto out;
    4085                 :            :         }
    4086                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
    4087                 :          0 :         trace_xhci_setup_device_slot(slot_ctx);
    4088                 :            : 
    4089         [ #  # ]:          0 :         if (setup == SETUP_CONTEXT_ONLY) {
    4090         [ #  # ]:          0 :                 if (GET_SLOT_STATE(le32_to_cpu(slot_ctx->dev_state)) ==
    4091                 :            :                     SLOT_STATE_DEFAULT) {
    4092                 :          0 :                         xhci_dbg(xhci, "Slot already in default state\n");
    4093                 :          0 :                         goto out;
    4094                 :            :                 }
    4095                 :            :         }
    4096                 :            : 
    4097                 :          0 :         command = xhci_alloc_command(xhci, true, GFP_KERNEL);
    4098         [ #  # ]:          0 :         if (!command) {
    4099                 :          0 :                 ret = -ENOMEM;
    4100                 :          0 :                 goto out;
    4101                 :            :         }
    4102                 :            : 
    4103                 :          0 :         command->in_ctx = virt_dev->in_ctx;
    4104                 :            : 
    4105                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->in_ctx);
    4106                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(virt_dev->in_ctx);
    4107         [ #  # ]:          0 :         if (!ctrl_ctx) {
    4108                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    4109                 :            :                                 __func__);
    4110                 :          0 :                 ret = -EINVAL;
    4111                 :          0 :                 goto out;
    4112                 :            :         }
    4113                 :            :         /*
    4114                 :            :          * If this is the first Set Address since device plug-in or
    4115                 :            :          * virt_device realloaction after a resume with an xHCI power loss,
    4116                 :            :          * then set up the slot context.
    4117                 :            :          */
    4118         [ #  # ]:          0 :         if (!slot_ctx->dev_info)
    4119                 :          0 :                 xhci_setup_addressable_virt_dev(xhci, udev);
    4120                 :            :         /* Otherwise, update the control endpoint ring enqueue pointer. */
    4121                 :            :         else
    4122                 :          0 :                 xhci_copy_ep0_dequeue_into_input_ctx(xhci, udev);
    4123                 :          0 :         ctrl_ctx->add_flags = cpu_to_le32(SLOT_FLAG | EP0_FLAG);
    4124                 :          0 :         ctrl_ctx->drop_flags = 0;
    4125                 :            : 
    4126                 :          0 :         trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
    4127                 :          0 :                                 le32_to_cpu(slot_ctx->dev_info) >> 27);
    4128                 :            : 
    4129                 :          0 :         trace_xhci_address_ctrl_ctx(ctrl_ctx);
    4130                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    4131                 :          0 :         trace_xhci_setup_device(virt_dev);
    4132                 :          0 :         ret = xhci_queue_address_device(xhci, command, virt_dev->in_ctx->dma,
    4133                 :          0 :                                         udev->slot_id, setup);
    4134         [ #  # ]:          0 :         if (ret) {
    4135                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    4136                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
    4137                 :            :                                 "FIXME: allocate a command ring segment");
    4138                 :          0 :                 goto out;
    4139                 :            :         }
    4140                 :          0 :         xhci_ring_cmd_db(xhci);
    4141                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    4142                 :            : 
    4143                 :            :         /* ctrl tx can take up to 5 sec; XXX: need more time for xHC? */
    4144                 :          0 :         wait_for_completion(command->completion);
    4145                 :            : 
    4146                 :            :         /* FIXME: From section 4.3.4: "Software shall be responsible for timing
    4147                 :            :          * the SetAddress() "recovery interval" required by USB and aborting the
    4148                 :            :          * command on a timeout.
    4149                 :            :          */
    4150   [ #  #  #  #  :          0 :         switch (command->status) {
                   #  # ]
    4151                 :          0 :         case COMP_COMMAND_ABORTED:
    4152                 :            :         case COMP_COMMAND_RING_STOPPED:
    4153                 :          0 :                 xhci_warn(xhci, "Timeout while waiting for setup device command\n");
    4154                 :          0 :                 ret = -ETIME;
    4155                 :          0 :                 break;
    4156                 :          0 :         case COMP_CONTEXT_STATE_ERROR:
    4157                 :            :         case COMP_SLOT_NOT_ENABLED_ERROR:
    4158                 :          0 :                 xhci_err(xhci, "Setup ERROR: setup %s command for slot %d.\n",
    4159                 :            :                          act, udev->slot_id);
    4160                 :          0 :                 ret = -EINVAL;
    4161                 :          0 :                 break;
    4162                 :          0 :         case COMP_USB_TRANSACTION_ERROR:
    4163                 :          0 :                 dev_warn(&udev->dev, "Device not responding to setup %s.\n", act);
    4164                 :            : 
    4165                 :          0 :                 mutex_unlock(&xhci->mutex);
    4166                 :          0 :                 ret = xhci_disable_slot(xhci, udev->slot_id);
    4167         [ #  # ]:          0 :                 if (!ret)
    4168                 :          0 :                         xhci_alloc_dev(hcd, udev);
    4169                 :          0 :                 kfree(command->completion);
    4170                 :          0 :                 kfree(command);
    4171                 :          0 :                 return -EPROTO;
    4172                 :          0 :         case COMP_INCOMPATIBLE_DEVICE_ERROR:
    4173                 :          0 :                 dev_warn(&udev->dev,
    4174                 :            :                          "ERROR: Incompatible device for setup %s command\n", act);
    4175                 :          0 :                 ret = -ENODEV;
    4176                 :          0 :                 break;
    4177                 :          0 :         case COMP_SUCCESS:
    4178                 :          0 :                 xhci_dbg_trace(xhci, trace_xhci_dbg_address,
    4179                 :            :                                "Successful setup %s command", act);
    4180                 :          0 :                 break;
    4181                 :          0 :         default:
    4182                 :          0 :                 xhci_err(xhci,
    4183                 :            :                          "ERROR: unexpected setup %s command completion code 0x%x.\n",
    4184                 :            :                          act, command->status);
    4185                 :          0 :                 trace_xhci_address_ctx(xhci, virt_dev->out_ctx, 1);
    4186                 :          0 :                 ret = -EINVAL;
    4187                 :          0 :                 break;
    4188                 :            :         }
    4189                 :          0 :         if (ret)
    4190                 :          0 :                 goto out;
    4191                 :          0 :         temp_64 = xhci_read_64(xhci, &xhci->op_regs->dcbaa_ptr);
    4192                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
    4193                 :            :                         "Op regs DCBAA ptr = %#016llx", temp_64);
    4194                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
    4195                 :            :                 "Slot ID %d dcbaa entry @%p = %#016llx",
    4196                 :            :                 udev->slot_id,
    4197                 :            :                 &xhci->dcbaa->dev_context_ptrs[udev->slot_id],
    4198                 :            :                 (unsigned long long)
    4199                 :          0 :                 le64_to_cpu(xhci->dcbaa->dev_context_ptrs[udev->slot_id]));
    4200                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
    4201                 :            :                         "Output Context DMA address = %#08llx",
    4202                 :          0 :                         (unsigned long long)virt_dev->out_ctx->dma);
    4203                 :          0 :         trace_xhci_address_ctx(xhci, virt_dev->in_ctx,
    4204                 :          0 :                                 le32_to_cpu(slot_ctx->dev_info) >> 27);
    4205                 :            :         /*
    4206                 :            :          * USB core uses address 1 for the roothubs, so we add one to the
    4207                 :            :          * address given back to us by the HC.
    4208                 :            :          */
    4209                 :          0 :         trace_xhci_address_ctx(xhci, virt_dev->out_ctx,
    4210                 :          0 :                                 le32_to_cpu(slot_ctx->dev_info) >> 27);
    4211                 :            :         /* Zero the input context control for later use */
    4212                 :          0 :         ctrl_ctx->add_flags = 0;
    4213                 :          0 :         ctrl_ctx->drop_flags = 0;
    4214                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, virt_dev->out_ctx);
    4215                 :          0 :         udev->devaddr = (u8)(le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
    4216                 :            : 
    4217                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_address,
    4218                 :            :                        "Internal device address = %d",
    4219                 :          0 :                        le32_to_cpu(slot_ctx->dev_state) & DEV_ADDR_MASK);
    4220                 :          0 : out:
    4221                 :          0 :         mutex_unlock(&xhci->mutex);
    4222         [ #  # ]:          0 :         if (command) {
    4223                 :          0 :                 kfree(command->completion);
    4224                 :          0 :                 kfree(command);
    4225                 :            :         }
    4226                 :            :         return ret;
    4227                 :            : }
    4228                 :            : 
    4229                 :          0 : static int xhci_address_device(struct usb_hcd *hcd, struct usb_device *udev)
    4230                 :            : {
    4231                 :          0 :         return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ADDRESS);
    4232                 :            : }
    4233                 :            : 
    4234                 :          0 : static int xhci_enable_device(struct usb_hcd *hcd, struct usb_device *udev)
    4235                 :            : {
    4236                 :          0 :         return xhci_setup_device(hcd, udev, SETUP_CONTEXT_ONLY);
    4237                 :            : }
    4238                 :            : 
    4239                 :            : /*
    4240                 :            :  * Transfer the port index into real index in the HW port status
    4241                 :            :  * registers. Caculate offset between the port's PORTSC register
    4242                 :            :  * and port status base. Divide the number of per port register
    4243                 :            :  * to get the real index. The raw port number bases 1.
    4244                 :            :  */
    4245                 :          0 : int xhci_find_raw_port_number(struct usb_hcd *hcd, int port1)
    4246                 :            : {
    4247                 :          0 :         struct xhci_hub *rhub;
    4248                 :            : 
    4249                 :          0 :         rhub = xhci_get_rhub(hcd);
    4250                 :          0 :         return rhub->ports[port1 - 1]->hw_portnum + 1;
    4251                 :            : }
    4252                 :            : 
    4253                 :            : /*
    4254                 :            :  * Issue an Evaluate Context command to change the Maximum Exit Latency in the
    4255                 :            :  * slot context.  If that succeeds, store the new MEL in the xhci_virt_device.
    4256                 :            :  */
    4257                 :          0 : static int __maybe_unused xhci_change_max_exit_latency(struct xhci_hcd *xhci,
    4258                 :            :                         struct usb_device *udev, u16 max_exit_latency)
    4259                 :            : {
    4260                 :          0 :         struct xhci_virt_device *virt_dev;
    4261                 :          0 :         struct xhci_command *command;
    4262                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    4263                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    4264                 :          0 :         unsigned long flags;
    4265                 :          0 :         int ret;
    4266                 :            : 
    4267                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    4268                 :            : 
    4269                 :          0 :         virt_dev = xhci->devs[udev->slot_id];
    4270                 :            : 
    4271                 :            :         /*
    4272                 :            :          * virt_dev might not exists yet if xHC resumed from hibernate (S4) and
    4273                 :            :          * xHC was re-initialized. Exit latency will be set later after
    4274                 :            :          * hub_port_finish_reset() is done and xhci->devs[] are re-allocated
    4275                 :            :          */
    4276                 :            : 
    4277   [ #  #  #  # ]:          0 :         if (!virt_dev || max_exit_latency == virt_dev->current_mel) {
    4278                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    4279                 :          0 :                 return 0;
    4280                 :            :         }
    4281                 :            : 
    4282                 :            :         /* Attempt to issue an Evaluate Context command to change the MEL. */
    4283                 :          0 :         command = xhci->lpm_command;
    4284                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(command->in_ctx);
    4285         [ #  # ]:          0 :         if (!ctrl_ctx) {
    4286                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    4287                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    4288                 :            :                                 __func__);
    4289                 :          0 :                 return -ENOMEM;
    4290                 :            :         }
    4291                 :            : 
    4292                 :          0 :         xhci_slot_copy(xhci, command->in_ctx, virt_dev->out_ctx);
    4293                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    4294                 :            : 
    4295                 :          0 :         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
    4296                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, command->in_ctx);
    4297                 :          0 :         slot_ctx->dev_info2 &= cpu_to_le32(~((u32) MAX_EXIT));
    4298                 :          0 :         slot_ctx->dev_info2 |= cpu_to_le32(max_exit_latency);
    4299                 :          0 :         slot_ctx->dev_state = 0;
    4300                 :            : 
    4301                 :          0 :         xhci_dbg_trace(xhci, trace_xhci_dbg_context_change,
    4302                 :            :                         "Set up evaluate context for LPM MEL change.");
    4303                 :            : 
    4304                 :            :         /* Issue and wait for the evaluate context command. */
    4305                 :          0 :         ret = xhci_configure_endpoint(xhci, udev, command,
    4306                 :            :                         true, true);
    4307                 :            : 
    4308         [ #  # ]:          0 :         if (!ret) {
    4309                 :          0 :                 spin_lock_irqsave(&xhci->lock, flags);
    4310                 :          0 :                 virt_dev->current_mel = max_exit_latency;
    4311                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    4312                 :            :         }
    4313                 :            :         return ret;
    4314                 :            : }
    4315                 :            : 
    4316                 :            : #ifdef CONFIG_PM
    4317                 :            : 
    4318                 :            : /* BESL to HIRD Encoding array for USB2 LPM */
    4319                 :            : static int xhci_besl_encoding[16] = {125, 150, 200, 300, 400, 500, 1000, 2000,
    4320                 :            :         3000, 4000, 5000, 6000, 7000, 8000, 9000, 10000};
    4321                 :            : 
    4322                 :            : /* Calculate HIRD/BESL for USB2 PORTPMSC*/
    4323                 :            : static int xhci_calculate_hird_besl(struct xhci_hcd *xhci,
    4324                 :            :                                         struct usb_device *udev)
    4325                 :            : {
    4326                 :            :         int u2del, besl, besl_host;
    4327                 :            :         int besl_device = 0;
    4328                 :            :         u32 field;
    4329                 :            : 
    4330                 :            :         u2del = HCS_U2_LATENCY(xhci->hcs_params3);
    4331                 :            :         field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
    4332                 :            : 
    4333                 :            :         if (field & USB_BESL_SUPPORT) {
    4334                 :            :                 for (besl_host = 0; besl_host < 16; besl_host++) {
    4335                 :            :                         if (xhci_besl_encoding[besl_host] >= u2del)
    4336                 :            :                                 break;
    4337                 :            :                 }
    4338                 :            :                 /* Use baseline BESL value as default */
    4339                 :            :                 if (field & USB_BESL_BASELINE_VALID)
    4340                 :            :                         besl_device = USB_GET_BESL_BASELINE(field);
    4341                 :            :                 else if (field & USB_BESL_DEEP_VALID)
    4342                 :            :                         besl_device = USB_GET_BESL_DEEP(field);
    4343                 :            :         } else {
    4344                 :            :                 if (u2del <= 50)
    4345                 :            :                         besl_host = 0;
    4346                 :            :                 else
    4347                 :            :                         besl_host = (u2del - 51) / 75 + 1;
    4348                 :            :         }
    4349                 :            : 
    4350                 :            :         besl = besl_host + besl_device;
    4351                 :            :         if (besl > 15)
    4352                 :            :                 besl = 15;
    4353                 :            : 
    4354                 :            :         return besl;
    4355                 :            : }
    4356                 :            : 
    4357                 :            : /* Calculate BESLD, L1 timeout and HIRDM for USB2 PORTHLPMC */
    4358                 :          0 : static int xhci_calculate_usb2_hw_lpm_params(struct usb_device *udev)
    4359                 :            : {
    4360                 :          0 :         u32 field;
    4361                 :          0 :         int l1;
    4362                 :          0 :         int besld = 0;
    4363                 :          0 :         int hirdm = 0;
    4364                 :            : 
    4365                 :          0 :         field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
    4366                 :            : 
    4367                 :            :         /* xHCI l1 is set in steps of 256us, xHCI 1.0 section 5.4.11.2 */
    4368                 :          0 :         l1 = udev->l1_params.timeout / 256;
    4369                 :            : 
    4370                 :            :         /* device has preferred BESLD */
    4371                 :          0 :         if (field & USB_BESL_DEEP_VALID) {
    4372                 :          0 :                 besld = USB_GET_BESL_DEEP(field);
    4373                 :          0 :                 hirdm = 1;
    4374                 :            :         }
    4375                 :            : 
    4376                 :          0 :         return PORT_BESLD(besld) | PORT_L1_TIMEOUT(l1) | PORT_HIRDM(hirdm);
    4377                 :            : }
    4378                 :            : 
    4379                 :          0 : static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
    4380                 :            :                         struct usb_device *udev, int enable)
    4381                 :            : {
    4382                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    4383                 :          0 :         struct xhci_port **ports;
    4384                 :          0 :         __le32 __iomem  *pm_addr, *hlpm_addr;
    4385                 :          0 :         u32             pm_val, hlpm_val, field;
    4386                 :          0 :         unsigned int    port_num;
    4387                 :          0 :         unsigned long   flags;
    4388                 :          0 :         int             hird, exit_latency;
    4389                 :          0 :         int             ret;
    4390                 :            : 
    4391   [ #  #  #  # ]:          0 :         if (hcd->speed >= HCD_USB3 || !xhci->hw_lpm_support ||
    4392         [ #  # ]:          0 :                         !udev->lpm_capable)
    4393                 :            :                 return -EPERM;
    4394                 :            : 
    4395   [ #  #  #  # ]:          0 :         if (!udev->parent || udev->parent->parent ||
    4396         [ #  # ]:          0 :                         udev->descriptor.bDeviceClass == USB_CLASS_HUB)
    4397                 :            :                 return -EPERM;
    4398                 :            : 
    4399         [ #  # ]:          0 :         if (udev->usb2_hw_lpm_capable != 1)
    4400                 :            :                 return -EPERM;
    4401                 :            : 
    4402                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    4403                 :            : 
    4404                 :          0 :         ports = xhci->usb2_rhub.ports;
    4405                 :          0 :         port_num = udev->portnum - 1;
    4406                 :          0 :         pm_addr = ports[port_num]->addr + PORTPMSC;
    4407                 :          0 :         pm_val = readl(pm_addr);
    4408                 :          0 :         hlpm_addr = ports[port_num]->addr + PORTHLPMC;
    4409                 :            : 
    4410                 :          0 :         xhci_dbg(xhci, "%s port %d USB2 hardware LPM\n",
    4411                 :            :                         enable ? "enable" : "disable", port_num + 1);
    4412                 :            : 
    4413   [ #  #  #  # ]:          0 :         if (enable && !(xhci->quirks & XHCI_HW_LPM_DISABLE)) {
    4414                 :            :                 /* Host supports BESL timeout instead of HIRD */
    4415         [ #  # ]:          0 :                 if (udev->usb2_hw_lpm_besl_capable) {
    4416                 :            :                         /* if device doesn't have a preferred BESL value use a
    4417                 :            :                          * default one which works with mixed HIRD and BESL
    4418                 :            :                          * systems. See XHCI_DEFAULT_BESL definition in xhci.h
    4419                 :            :                          */
    4420                 :          0 :                         field = le32_to_cpu(udev->bos->ext_cap->bmAttributes);
    4421         [ #  # ]:          0 :                         if ((field & USB_BESL_SUPPORT) &&
    4422                 :            :                             (field & USB_BESL_BASELINE_VALID))
    4423                 :          0 :                                 hird = USB_GET_BESL_BASELINE(field);
    4424                 :            :                         else
    4425                 :          0 :                                 hird = udev->l1_params.besl;
    4426                 :            : 
    4427                 :          0 :                         exit_latency = xhci_besl_encoding[hird];
    4428                 :          0 :                         spin_unlock_irqrestore(&xhci->lock, flags);
    4429                 :            : 
    4430                 :            :                         /* USB 3.0 code dedicate one xhci->lpm_command->in_ctx
    4431                 :            :                          * input context for link powermanagement evaluate
    4432                 :            :                          * context commands. It is protected by hcd->bandwidth
    4433                 :            :                          * mutex and is shared by all devices. We need to set
    4434                 :            :                          * the max ext latency in USB 2 BESL LPM as well, so
    4435                 :            :                          * use the same mutex and xhci_change_max_exit_latency()
    4436                 :            :                          */
    4437                 :          0 :                         mutex_lock(hcd->bandwidth_mutex);
    4438                 :          0 :                         ret = xhci_change_max_exit_latency(xhci, udev,
    4439                 :            :                                                            exit_latency);
    4440                 :          0 :                         mutex_unlock(hcd->bandwidth_mutex);
    4441                 :            : 
    4442         [ #  # ]:          0 :                         if (ret < 0)
    4443                 :            :                                 return ret;
    4444                 :          0 :                         spin_lock_irqsave(&xhci->lock, flags);
    4445                 :            : 
    4446         [ #  # ]:          0 :                         hlpm_val = xhci_calculate_usb2_hw_lpm_params(udev);
    4447                 :          0 :                         writel(hlpm_val, hlpm_addr);
    4448                 :            :                         /* flush write */
    4449                 :          0 :                         readl(hlpm_addr);
    4450                 :            :                 } else {
    4451                 :          0 :                         hird = xhci_calculate_hird_besl(xhci, udev);
    4452                 :            :                 }
    4453                 :            : 
    4454                 :          0 :                 pm_val &= ~PORT_HIRD_MASK;
    4455                 :          0 :                 pm_val |= PORT_HIRD(hird) | PORT_RWE | PORT_L1DS(udev->slot_id);
    4456                 :          0 :                 writel(pm_val, pm_addr);
    4457                 :          0 :                 pm_val = readl(pm_addr);
    4458                 :          0 :                 pm_val |= PORT_HLE;
    4459                 :          0 :                 writel(pm_val, pm_addr);
    4460                 :            :                 /* flush write */
    4461                 :          0 :                 readl(pm_addr);
    4462                 :            :         } else {
    4463                 :          0 :                 pm_val &= ~(PORT_HLE | PORT_RWE | PORT_HIRD_MASK | PORT_L1DS_MASK);
    4464                 :          0 :                 writel(pm_val, pm_addr);
    4465                 :            :                 /* flush write */
    4466                 :          0 :                 readl(pm_addr);
    4467         [ #  # ]:          0 :                 if (udev->usb2_hw_lpm_besl_capable) {
    4468                 :          0 :                         spin_unlock_irqrestore(&xhci->lock, flags);
    4469                 :          0 :                         mutex_lock(hcd->bandwidth_mutex);
    4470                 :          0 :                         xhci_change_max_exit_latency(xhci, udev, 0);
    4471                 :          0 :                         mutex_unlock(hcd->bandwidth_mutex);
    4472                 :          0 :                         return 0;
    4473                 :            :                 }
    4474                 :            :         }
    4475                 :            : 
    4476                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    4477                 :          0 :         return 0;
    4478                 :            : }
    4479                 :            : 
    4480                 :            : /* check if a usb2 port supports a given extened capability protocol
    4481                 :            :  * only USB2 ports extended protocol capability values are cached.
    4482                 :            :  * Return 1 if capability is supported
    4483                 :            :  */
    4484                 :          0 : static int xhci_check_usb2_port_capability(struct xhci_hcd *xhci, int port,
    4485                 :            :                                            unsigned capability)
    4486                 :            : {
    4487                 :            :         u32 port_offset, port_count;
    4488                 :            :         int i;
    4489                 :            : 
    4490   [ #  #  #  # ]:          0 :         for (i = 0; i < xhci->num_ext_caps; i++) {
    4491   [ #  #  #  # ]:          0 :                 if (xhci->ext_caps[i] & capability) {
    4492                 :            :                         /* port offsets starts at 1 */
    4493                 :          0 :                         port_offset = XHCI_EXT_PORT_OFF(xhci->ext_caps[i]) - 1;
    4494                 :          0 :                         port_count = XHCI_EXT_PORT_COUNT(xhci->ext_caps[i]);
    4495   [ #  #  #  # ]:          0 :                         if (port >= port_offset &&
    4496   [ #  #  #  # ]:          0 :                             port < port_offset + port_count)
    4497                 :            :                                 return 1;
    4498                 :            :                 }
    4499                 :            :         }
    4500                 :            :         return 0;
    4501                 :            : }
    4502                 :            : 
    4503                 :          0 : static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
    4504                 :            : {
    4505                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    4506                 :          0 :         int             portnum = udev->portnum - 1;
    4507                 :            : 
    4508   [ #  #  #  # ]:          0 :         if (hcd->speed >= HCD_USB3 || !udev->lpm_capable)
    4509                 :            :                 return 0;
    4510                 :            : 
    4511                 :            :         /* we only support lpm for non-hub device connected to root hub yet */
    4512   [ #  #  #  # ]:          0 :         if (!udev->parent || udev->parent->parent ||
    4513         [ #  # ]:          0 :                         udev->descriptor.bDeviceClass == USB_CLASS_HUB)
    4514                 :            :                 return 0;
    4515                 :            : 
    4516   [ #  #  #  # ]:          0 :         if (xhci->hw_lpm_support == 1 &&
    4517                 :          0 :                         xhci_check_usb2_port_capability(
    4518                 :            :                                 xhci, portnum, XHCI_HLC)) {
    4519                 :          0 :                 udev->usb2_hw_lpm_capable = 1;
    4520                 :          0 :                 udev->l1_params.timeout = XHCI_L1_TIMEOUT;
    4521                 :          0 :                 udev->l1_params.besl = XHCI_DEFAULT_BESL;
    4522         [ #  # ]:          0 :                 if (xhci_check_usb2_port_capability(xhci, portnum,
    4523                 :            :                                         XHCI_BLC))
    4524                 :          0 :                         udev->usb2_hw_lpm_besl_capable = 1;
    4525                 :            :         }
    4526                 :            : 
    4527                 :            :         return 0;
    4528                 :            : }
    4529                 :            : 
    4530                 :            : /*---------------------- USB 3.0 Link PM functions ------------------------*/
    4531                 :            : 
    4532                 :            : /* Service interval in nanoseconds = 2^(bInterval - 1) * 125us * 1000ns / 1us */
    4533                 :            : static unsigned long long xhci_service_interval_to_ns(
    4534                 :            :                 struct usb_endpoint_descriptor *desc)
    4535                 :            : {
    4536                 :            :         return (1ULL << (desc->bInterval - 1)) * 125 * 1000;
    4537                 :            : }
    4538                 :            : 
    4539                 :          0 : static u16 xhci_get_timeout_no_hub_lpm(struct usb_device *udev,
    4540                 :            :                 enum usb3_link_state state)
    4541                 :            : {
    4542                 :          0 :         unsigned long long sel;
    4543                 :          0 :         unsigned long long pel;
    4544                 :          0 :         unsigned int max_sel_pel;
    4545                 :          0 :         char *state_name;
    4546                 :            : 
    4547      [ #  #  # ]:          0 :         switch (state) {
    4548                 :          0 :         case USB3_LPM_U1:
    4549                 :            :                 /* Convert SEL and PEL stored in nanoseconds to microseconds */
    4550                 :          0 :                 sel = DIV_ROUND_UP(udev->u1_params.sel, 1000);
    4551                 :          0 :                 pel = DIV_ROUND_UP(udev->u1_params.pel, 1000);
    4552                 :          0 :                 max_sel_pel = USB3_LPM_MAX_U1_SEL_PEL;
    4553                 :          0 :                 state_name = "U1";
    4554                 :          0 :                 break;
    4555                 :          0 :         case USB3_LPM_U2:
    4556                 :          0 :                 sel = DIV_ROUND_UP(udev->u2_params.sel, 1000);
    4557                 :          0 :                 pel = DIV_ROUND_UP(udev->u2_params.pel, 1000);
    4558                 :          0 :                 max_sel_pel = USB3_LPM_MAX_U2_SEL_PEL;
    4559                 :          0 :                 state_name = "U2";
    4560                 :          0 :                 break;
    4561                 :          0 :         default:
    4562                 :          0 :                 dev_warn(&udev->dev, "%s: Can't get timeout for non-U1 or U2 state.\n",
    4563                 :            :                                 __func__);
    4564                 :          0 :                 return USB3_LPM_DISABLED;
    4565                 :            :         }
    4566                 :            : 
    4567   [ #  #  #  # ]:          0 :         if (sel <= max_sel_pel && pel <= max_sel_pel)
    4568                 :          0 :                 return USB3_LPM_DEVICE_INITIATED;
    4569                 :            : 
    4570                 :            :         if (sel > max_sel_pel)
    4571                 :            :                 dev_dbg(&udev->dev, "Device-initiated %s disabled "
    4572                 :            :                                 "due to long SEL %llu ms\n",
    4573                 :            :                                 state_name, sel);
    4574                 :            :         else
    4575                 :            :                 dev_dbg(&udev->dev, "Device-initiated %s disabled "
    4576                 :            :                                 "due to long PEL %llu ms\n",
    4577                 :            :                                 state_name, pel);
    4578                 :            :         return USB3_LPM_DISABLED;
    4579                 :            : }
    4580                 :            : 
    4581                 :            : /* The U1 timeout should be the maximum of the following values:
    4582                 :            :  *  - For control endpoints, U1 system exit latency (SEL) * 3
    4583                 :            :  *  - For bulk endpoints, U1 SEL * 5
    4584                 :            :  *  - For interrupt endpoints:
    4585                 :            :  *    - Notification EPs, U1 SEL * 3
    4586                 :            :  *    - Periodic EPs, max(105% of bInterval, U1 SEL * 2)
    4587                 :            :  *  - For isochronous endpoints, max(105% of bInterval, U1 SEL * 2)
    4588                 :            :  */
    4589                 :            : static unsigned long long xhci_calculate_intel_u1_timeout(
    4590                 :            :                 struct usb_device *udev,
    4591                 :            :                 struct usb_endpoint_descriptor *desc)
    4592                 :            : {
    4593                 :            :         unsigned long long timeout_ns;
    4594                 :            :         int ep_type;
    4595                 :            :         int intr_type;
    4596                 :            : 
    4597                 :            :         ep_type = usb_endpoint_type(desc);
    4598                 :            :         switch (ep_type) {
    4599                 :            :         case USB_ENDPOINT_XFER_CONTROL:
    4600                 :            :                 timeout_ns = udev->u1_params.sel * 3;
    4601                 :            :                 break;
    4602                 :            :         case USB_ENDPOINT_XFER_BULK:
    4603                 :            :                 timeout_ns = udev->u1_params.sel * 5;
    4604                 :            :                 break;
    4605                 :            :         case USB_ENDPOINT_XFER_INT:
    4606                 :            :                 intr_type = usb_endpoint_interrupt_type(desc);
    4607                 :            :                 if (intr_type == USB_ENDPOINT_INTR_NOTIFICATION) {
    4608                 :            :                         timeout_ns = udev->u1_params.sel * 3;
    4609                 :            :                         break;
    4610                 :            :                 }
    4611                 :            :                 /* Otherwise the calculation is the same as isoc eps */
    4612                 :            :                 /* fall through */
    4613                 :            :         case USB_ENDPOINT_XFER_ISOC:
    4614                 :            :                 timeout_ns = xhci_service_interval_to_ns(desc);
    4615                 :            :                 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns * 105, 100);
    4616                 :            :                 if (timeout_ns < udev->u1_params.sel * 2)
    4617                 :            :                         timeout_ns = udev->u1_params.sel * 2;
    4618                 :            :                 break;
    4619                 :            :         default:
    4620                 :            :                 return 0;
    4621                 :            :         }
    4622                 :            : 
    4623                 :            :         return timeout_ns;
    4624                 :            : }
    4625                 :            : 
    4626                 :            : /* Returns the hub-encoded U1 timeout value. */
    4627                 :            : static u16 xhci_calculate_u1_timeout(struct xhci_hcd *xhci,
    4628                 :            :                 struct usb_device *udev,
    4629                 :            :                 struct usb_endpoint_descriptor *desc)
    4630                 :            : {
    4631                 :            :         unsigned long long timeout_ns;
    4632                 :            : 
    4633                 :            :         /* Prevent U1 if service interval is shorter than U1 exit latency */
    4634                 :            :         if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
    4635                 :            :                 if (xhci_service_interval_to_ns(desc) <= udev->u1_params.mel) {
    4636                 :            :                         dev_dbg(&udev->dev, "Disable U1, ESIT shorter than exit latency\n");
    4637                 :            :                         return USB3_LPM_DISABLED;
    4638                 :            :                 }
    4639                 :            :         }
    4640                 :            : 
    4641                 :            :         if (xhci->quirks & XHCI_INTEL_HOST)
    4642                 :            :                 timeout_ns = xhci_calculate_intel_u1_timeout(udev, desc);
    4643                 :            :         else
    4644                 :            :                 timeout_ns = udev->u1_params.sel;
    4645                 :            : 
    4646                 :            :         /* The U1 timeout is encoded in 1us intervals.
    4647                 :            :          * Don't return a timeout of zero, because that's USB3_LPM_DISABLED.
    4648                 :            :          */
    4649                 :            :         if (timeout_ns == USB3_LPM_DISABLED)
    4650                 :            :                 timeout_ns = 1;
    4651                 :            :         else
    4652                 :            :                 timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 1000);
    4653                 :            : 
    4654                 :            :         /* If the necessary timeout value is bigger than what we can set in the
    4655                 :            :          * USB 3.0 hub, we have to disable hub-initiated U1.
    4656                 :            :          */
    4657                 :            :         if (timeout_ns <= USB3_LPM_U1_MAX_TIMEOUT)
    4658                 :            :                 return timeout_ns;
    4659                 :            :         dev_dbg(&udev->dev, "Hub-initiated U1 disabled "
    4660                 :            :                         "due to long timeout %llu ms\n", timeout_ns);
    4661                 :            :         return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U1);
    4662                 :            : }
    4663                 :            : 
    4664                 :            : /* The U2 timeout should be the maximum of:
    4665                 :            :  *  - 10 ms (to avoid the bandwidth impact on the scheduler)
    4666                 :            :  *  - largest bInterval of any active periodic endpoint (to avoid going
    4667                 :            :  *    into lower power link states between intervals).
    4668                 :            :  *  - the U2 Exit Latency of the device
    4669                 :            :  */
    4670                 :            : static unsigned long long xhci_calculate_intel_u2_timeout(
    4671                 :            :                 struct usb_device *udev,
    4672                 :            :                 struct usb_endpoint_descriptor *desc)
    4673                 :            : {
    4674                 :            :         unsigned long long timeout_ns;
    4675                 :            :         unsigned long long u2_del_ns;
    4676                 :            : 
    4677                 :            :         timeout_ns = 10 * 1000 * 1000;
    4678                 :            : 
    4679                 :            :         if ((usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) &&
    4680                 :            :                         (xhci_service_interval_to_ns(desc) > timeout_ns))
    4681                 :            :                 timeout_ns = xhci_service_interval_to_ns(desc);
    4682                 :            : 
    4683                 :            :         u2_del_ns = le16_to_cpu(udev->bos->ss_cap->bU2DevExitLat) * 1000ULL;
    4684                 :            :         if (u2_del_ns > timeout_ns)
    4685                 :            :                 timeout_ns = u2_del_ns;
    4686                 :            : 
    4687                 :            :         return timeout_ns;
    4688                 :            : }
    4689                 :            : 
    4690                 :            : /* Returns the hub-encoded U2 timeout value. */
    4691                 :            : static u16 xhci_calculate_u2_timeout(struct xhci_hcd *xhci,
    4692                 :            :                 struct usb_device *udev,
    4693                 :            :                 struct usb_endpoint_descriptor *desc)
    4694                 :            : {
    4695                 :            :         unsigned long long timeout_ns;
    4696                 :            : 
    4697                 :            :         /* Prevent U2 if service interval is shorter than U2 exit latency */
    4698                 :            :         if (usb_endpoint_xfer_int(desc) || usb_endpoint_xfer_isoc(desc)) {
    4699                 :            :                 if (xhci_service_interval_to_ns(desc) <= udev->u2_params.mel) {
    4700                 :            :                         dev_dbg(&udev->dev, "Disable U2, ESIT shorter than exit latency\n");
    4701                 :            :                         return USB3_LPM_DISABLED;
    4702                 :            :                 }
    4703                 :            :         }
    4704                 :            : 
    4705                 :            :         if (xhci->quirks & XHCI_INTEL_HOST)
    4706                 :            :                 timeout_ns = xhci_calculate_intel_u2_timeout(udev, desc);
    4707                 :            :         else
    4708                 :            :                 timeout_ns = udev->u2_params.sel;
    4709                 :            : 
    4710                 :            :         /* The U2 timeout is encoded in 256us intervals */
    4711                 :            :         timeout_ns = DIV_ROUND_UP_ULL(timeout_ns, 256 * 1000);
    4712                 :            :         /* If the necessary timeout value is bigger than what we can set in the
    4713                 :            :          * USB 3.0 hub, we have to disable hub-initiated U2.
    4714                 :            :          */
    4715                 :            :         if (timeout_ns <= USB3_LPM_U2_MAX_TIMEOUT)
    4716                 :            :                 return timeout_ns;
    4717                 :            :         dev_dbg(&udev->dev, "Hub-initiated U2 disabled "
    4718                 :            :                         "due to long timeout %llu ms\n", timeout_ns);
    4719                 :            :         return xhci_get_timeout_no_hub_lpm(udev, USB3_LPM_U2);
    4720                 :            : }
    4721                 :            : 
    4722                 :            : static u16 xhci_call_host_update_timeout_for_endpoint(struct xhci_hcd *xhci,
    4723                 :            :                 struct usb_device *udev,
    4724                 :            :                 struct usb_endpoint_descriptor *desc,
    4725                 :            :                 enum usb3_link_state state,
    4726                 :            :                 u16 *timeout)
    4727                 :            : {
    4728                 :            :         if (state == USB3_LPM_U1)
    4729                 :            :                 return xhci_calculate_u1_timeout(xhci, udev, desc);
    4730                 :            :         else if (state == USB3_LPM_U2)
    4731                 :            :                 return xhci_calculate_u2_timeout(xhci, udev, desc);
    4732                 :            : 
    4733                 :            :         return USB3_LPM_DISABLED;
    4734                 :            : }
    4735                 :            : 
    4736                 :          0 : static int xhci_update_timeout_for_endpoint(struct xhci_hcd *xhci,
    4737                 :            :                 struct usb_device *udev,
    4738                 :            :                 struct usb_endpoint_descriptor *desc,
    4739                 :            :                 enum usb3_link_state state,
    4740                 :            :                 u16 *timeout)
    4741                 :            : {
    4742                 :          0 :         u16 alt_timeout;
    4743                 :            : 
    4744                 :          0 :         alt_timeout = xhci_call_host_update_timeout_for_endpoint(xhci, udev,
    4745                 :            :                 desc, state, timeout);
    4746                 :            : 
    4747                 :            :         /* If we found we can't enable hub-initiated LPM, and
    4748                 :            :          * the U1 or U2 exit latency was too high to allow
    4749                 :            :          * device-initiated LPM as well, then we will disable LPM
    4750                 :            :          * for this device, so stop searching any further.
    4751                 :            :          */
    4752         [ #  # ]:          0 :         if (alt_timeout == USB3_LPM_DISABLED) {
    4753                 :            :                 *timeout = alt_timeout;
    4754                 :            :                 return -E2BIG;
    4755                 :            :         }
    4756         [ #  # ]:          0 :         if (alt_timeout > *timeout)
    4757                 :          0 :                 *timeout = alt_timeout;
    4758                 :            :         return 0;
    4759                 :            : }
    4760                 :            : 
    4761                 :            : static int xhci_update_timeout_for_interface(struct xhci_hcd *xhci,
    4762                 :            :                 struct usb_device *udev,
    4763                 :            :                 struct usb_host_interface *alt,
    4764                 :            :                 enum usb3_link_state state,
    4765                 :            :                 u16 *timeout)
    4766                 :            : {
    4767                 :            :         int j;
    4768                 :            : 
    4769                 :            :         for (j = 0; j < alt->desc.bNumEndpoints; j++) {
    4770                 :            :                 if (xhci_update_timeout_for_endpoint(xhci, udev,
    4771                 :            :                                         &alt->endpoint[j].desc, state, timeout))
    4772                 :            :                         return -E2BIG;
    4773                 :            :                 continue;
    4774                 :            :         }
    4775                 :            :         return 0;
    4776                 :            : }
    4777                 :            : 
    4778                 :          0 : static int xhci_check_intel_tier_policy(struct usb_device *udev,
    4779                 :            :                 enum usb3_link_state state)
    4780                 :            : {
    4781                 :          0 :         struct usb_device *parent;
    4782                 :          0 :         unsigned int num_hubs;
    4783                 :            : 
    4784                 :          0 :         if (state == USB3_LPM_U2)
    4785                 :            :                 return 0;
    4786                 :            : 
    4787                 :            :         /* Don't enable U1 if the device is on a 2nd tier hub or lower. */
    4788         [ #  # ]:          0 :         for (parent = udev->parent, num_hubs = 0; parent->parent;
    4789                 :          0 :                         parent = parent->parent)
    4790                 :          0 :                 num_hubs++;
    4791                 :            : 
    4792         [ #  # ]:          0 :         if (num_hubs < 2)
    4793                 :            :                 return 0;
    4794                 :            : 
    4795                 :            :         dev_dbg(&udev->dev, "Disabling U1 link state for device"
    4796                 :            :                         " below second-tier hub.\n");
    4797                 :            :         dev_dbg(&udev->dev, "Plug device into first-tier hub "
    4798                 :            :                         "to decrease power consumption.\n");
    4799                 :            :         return -E2BIG;
    4800                 :            : }
    4801                 :            : 
    4802                 :          0 : static int xhci_check_tier_policy(struct xhci_hcd *xhci,
    4803                 :            :                 struct usb_device *udev,
    4804                 :            :                 enum usb3_link_state state)
    4805                 :            : {
    4806                 :          0 :         if (xhci->quirks & XHCI_INTEL_HOST)
    4807         [ #  # ]:          0 :                 return xhci_check_intel_tier_policy(udev, state);
    4808                 :            :         else
    4809                 :            :                 return 0;
    4810                 :            : }
    4811                 :            : 
    4812                 :            : /* Returns the U1 or U2 timeout that should be enabled.
    4813                 :            :  * If the tier check or timeout setting functions return with a non-zero exit
    4814                 :            :  * code, that means the timeout value has been finalized and we shouldn't look
    4815                 :            :  * at any more endpoints.
    4816                 :            :  */
    4817                 :          0 : static u16 xhci_calculate_lpm_timeout(struct usb_hcd *hcd,
    4818                 :            :                         struct usb_device *udev, enum usb3_link_state state)
    4819                 :            : {
    4820                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    4821                 :          0 :         struct usb_host_config *config;
    4822                 :          0 :         char *state_name;
    4823                 :          0 :         int i;
    4824                 :          0 :         u16 timeout = USB3_LPM_DISABLED;
    4825                 :            : 
    4826         [ #  # ]:          0 :         if (state == USB3_LPM_U1)
    4827                 :            :                 state_name = "U1";
    4828         [ #  # ]:          0 :         else if (state == USB3_LPM_U2)
    4829                 :            :                 state_name = "U2";
    4830                 :            :         else {
    4831                 :          0 :                 dev_warn(&udev->dev, "Can't enable unknown link state %i\n",
    4832                 :            :                                 state);
    4833                 :          0 :                 return timeout;
    4834                 :            :         }
    4835                 :            : 
    4836         [ #  # ]:          0 :         if (xhci_check_tier_policy(xhci, udev, state) < 0)
    4837                 :            :                 return timeout;
    4838                 :            : 
    4839                 :            :         /* Gather some information about the currently installed configuration
    4840                 :            :          * and alternate interface settings.
    4841                 :            :          */
    4842                 :          0 :         if (xhci_update_timeout_for_endpoint(xhci, udev, &udev->ep0.desc,
    4843                 :            :                         state, &timeout))
    4844                 :            :                 return timeout;
    4845                 :            : 
    4846                 :          0 :         config = udev->actconfig;
    4847         [ #  # ]:          0 :         if (!config)
    4848                 :          0 :                 return timeout;
    4849                 :            : 
    4850         [ #  # ]:          0 :         for (i = 0; i < config->desc.bNumInterfaces; i++) {
    4851                 :          0 :                 struct usb_driver *driver;
    4852                 :          0 :                 struct usb_interface *intf = config->interface[i];
    4853                 :            : 
    4854         [ #  # ]:          0 :                 if (!intf)
    4855                 :          0 :                         continue;
    4856                 :            : 
    4857                 :            :                 /* Check if any currently bound drivers want hub-initiated LPM
    4858                 :            :                  * disabled.
    4859                 :            :                  */
    4860         [ #  # ]:          0 :                 if (intf->dev.driver) {
    4861                 :          0 :                         driver = to_usb_driver(intf->dev.driver);
    4862   [ #  #  #  # ]:          0 :                         if (driver && driver->disable_hub_initiated_lpm) {
    4863                 :          0 :                                 dev_dbg(&udev->dev, "Hub-initiated %s disabled at request of driver %s\n",
    4864                 :            :                                         state_name, driver->name);
    4865                 :          0 :                                 timeout = xhci_get_timeout_no_hub_lpm(udev,
    4866                 :            :                                                                       state);
    4867         [ #  # ]:          0 :                                 if (timeout == USB3_LPM_DISABLED)
    4868                 :            :                                         return timeout;
    4869                 :            :                         }
    4870                 :            :                 }
    4871                 :            : 
    4872                 :            :                 /* Not sure how this could happen... */
    4873         [ #  # ]:          0 :                 if (!intf->cur_altsetting)
    4874                 :          0 :                         continue;
    4875                 :            : 
    4876         [ #  # ]:          0 :                 if (xhci_update_timeout_for_interface(xhci, udev,
    4877                 :            :                                         intf->cur_altsetting,
    4878                 :            :                                         state, &timeout))
    4879                 :          0 :                         return timeout;
    4880                 :            :         }
    4881                 :          0 :         return timeout;
    4882                 :            : }
    4883                 :            : 
    4884                 :          0 : static int calculate_max_exit_latency(struct usb_device *udev,
    4885                 :            :                 enum usb3_link_state state_changed,
    4886                 :            :                 u16 hub_encoded_timeout)
    4887                 :            : {
    4888                 :          0 :         unsigned long long u1_mel_us = 0;
    4889                 :          0 :         unsigned long long u2_mel_us = 0;
    4890                 :          0 :         unsigned long long mel_us = 0;
    4891                 :          0 :         bool disabling_u1;
    4892                 :          0 :         bool disabling_u2;
    4893                 :          0 :         bool enabling_u1;
    4894                 :          0 :         bool enabling_u2;
    4895                 :            : 
    4896                 :          0 :         disabling_u1 = (state_changed == USB3_LPM_U1 &&
    4897                 :          0 :                         hub_encoded_timeout == USB3_LPM_DISABLED);
    4898                 :          0 :         disabling_u2 = (state_changed == USB3_LPM_U2 &&
    4899                 :            :                         hub_encoded_timeout == USB3_LPM_DISABLED);
    4900                 :            : 
    4901                 :          0 :         enabling_u1 = (state_changed == USB3_LPM_U1 &&
    4902                 :          0 :                         hub_encoded_timeout != USB3_LPM_DISABLED);
    4903                 :          0 :         enabling_u2 = (state_changed == USB3_LPM_U2 &&
    4904                 :            :                         hub_encoded_timeout != USB3_LPM_DISABLED);
    4905                 :            : 
    4906                 :            :         /* If U1 was already enabled and we're not disabling it,
    4907                 :            :          * or we're going to enable U1, account for the U1 max exit latency.
    4908                 :            :          */
    4909   [ #  #  #  #  :          0 :         if ((udev->u1_params.timeout != USB3_LPM_DISABLED && !disabling_u1) ||
                   #  # ]
    4910                 :            :                         enabling_u1)
    4911                 :          0 :                 u1_mel_us = DIV_ROUND_UP(udev->u1_params.mel, 1000);
    4912   [ #  #  #  #  :          0 :         if ((udev->u2_params.timeout != USB3_LPM_DISABLED && !disabling_u2) ||
                   #  # ]
    4913                 :            :                         enabling_u2)
    4914                 :          0 :                 u2_mel_us = DIV_ROUND_UP(udev->u2_params.mel, 1000);
    4915                 :            : 
    4916                 :          0 :         if (u1_mel_us > u2_mel_us)
    4917                 :            :                 mel_us = u1_mel_us;
    4918                 :            :         else
    4919                 :            :                 mel_us = u2_mel_us;
    4920                 :            :         /* xHCI host controller max exit latency field is only 16 bits wide. */
    4921         [ #  # ]:          0 :         if (mel_us > MAX_EXIT) {
    4922                 :          0 :                 dev_warn(&udev->dev, "Link PM max exit latency of %lluus "
    4923                 :            :                                 "is too big.\n", mel_us);
    4924                 :          0 :                 return -E2BIG;
    4925                 :            :         }
    4926                 :          0 :         return mel_us;
    4927                 :            : }
    4928                 :            : 
    4929                 :            : /* Returns the USB3 hub-encoded value for the U1/U2 timeout. */
    4930                 :          0 : static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
    4931                 :            :                         struct usb_device *udev, enum usb3_link_state state)
    4932                 :            : {
    4933                 :          0 :         struct xhci_hcd *xhci;
    4934                 :          0 :         u16 hub_encoded_timeout;
    4935                 :          0 :         int mel;
    4936                 :          0 :         int ret;
    4937                 :            : 
    4938                 :          0 :         xhci = hcd_to_xhci(hcd);
    4939                 :            :         /* The LPM timeout values are pretty host-controller specific, so don't
    4940                 :            :          * enable hub-initiated timeouts unless the vendor has provided
    4941                 :            :          * information about their timeout algorithm.
    4942                 :            :          */
    4943   [ #  #  #  # ]:          0 :         if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
    4944         [ #  # ]:          0 :                         !xhci->devs[udev->slot_id])
    4945                 :            :                 return USB3_LPM_DISABLED;
    4946                 :            : 
    4947                 :          0 :         hub_encoded_timeout = xhci_calculate_lpm_timeout(hcd, udev, state);
    4948                 :          0 :         mel = calculate_max_exit_latency(udev, state, hub_encoded_timeout);
    4949         [ #  # ]:          0 :         if (mel < 0) {
    4950                 :            :                 /* Max Exit Latency is too big, disable LPM. */
    4951                 :          0 :                 hub_encoded_timeout = USB3_LPM_DISABLED;
    4952                 :          0 :                 mel = 0;
    4953                 :            :         }
    4954                 :            : 
    4955                 :          0 :         ret = xhci_change_max_exit_latency(xhci, udev, mel);
    4956         [ #  # ]:          0 :         if (ret)
    4957                 :            :                 return ret;
    4958                 :          0 :         return hub_encoded_timeout;
    4959                 :            : }
    4960                 :            : 
    4961                 :          0 : static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
    4962                 :            :                         struct usb_device *udev, enum usb3_link_state state)
    4963                 :            : {
    4964                 :          0 :         struct xhci_hcd *xhci;
    4965                 :          0 :         u16 mel;
    4966                 :            : 
    4967                 :          0 :         xhci = hcd_to_xhci(hcd);
    4968   [ #  #  #  # ]:          0 :         if (!xhci || !(xhci->quirks & XHCI_LPM_SUPPORT) ||
    4969         [ #  # ]:          0 :                         !xhci->devs[udev->slot_id])
    4970                 :            :                 return 0;
    4971                 :            : 
    4972                 :          0 :         mel = calculate_max_exit_latency(udev, state, USB3_LPM_DISABLED);
    4973                 :          0 :         return xhci_change_max_exit_latency(xhci, udev, mel);
    4974                 :            : }
    4975                 :            : #else /* CONFIG_PM */
    4976                 :            : 
    4977                 :            : static int xhci_set_usb2_hardware_lpm(struct usb_hcd *hcd,
    4978                 :            :                                 struct usb_device *udev, int enable)
    4979                 :            : {
    4980                 :            :         return 0;
    4981                 :            : }
    4982                 :            : 
    4983                 :            : static int xhci_update_device(struct usb_hcd *hcd, struct usb_device *udev)
    4984                 :            : {
    4985                 :            :         return 0;
    4986                 :            : }
    4987                 :            : 
    4988                 :            : static int xhci_enable_usb3_lpm_timeout(struct usb_hcd *hcd,
    4989                 :            :                         struct usb_device *udev, enum usb3_link_state state)
    4990                 :            : {
    4991                 :            :         return USB3_LPM_DISABLED;
    4992                 :            : }
    4993                 :            : 
    4994                 :            : static int xhci_disable_usb3_lpm_timeout(struct usb_hcd *hcd,
    4995                 :            :                         struct usb_device *udev, enum usb3_link_state state)
    4996                 :            : {
    4997                 :            :         return 0;
    4998                 :            : }
    4999                 :            : #endif  /* CONFIG_PM */
    5000                 :            : 
    5001                 :            : /*-------------------------------------------------------------------------*/
    5002                 :            : 
    5003                 :            : /* Once a hub descriptor is fetched for a device, we need to update the xHC's
    5004                 :            :  * internal data structures for the device.
    5005                 :            :  */
    5006                 :          0 : static int xhci_update_hub_device(struct usb_hcd *hcd, struct usb_device *hdev,
    5007                 :            :                         struct usb_tt *tt, gfp_t mem_flags)
    5008                 :            : {
    5009                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    5010                 :          0 :         struct xhci_virt_device *vdev;
    5011                 :          0 :         struct xhci_command *config_cmd;
    5012                 :          0 :         struct xhci_input_control_ctx *ctrl_ctx;
    5013                 :          0 :         struct xhci_slot_ctx *slot_ctx;
    5014                 :          0 :         unsigned long flags;
    5015                 :          0 :         unsigned think_time;
    5016                 :          0 :         int ret;
    5017                 :            : 
    5018                 :            :         /* Ignore root hubs */
    5019         [ #  # ]:          0 :         if (!hdev->parent)
    5020                 :            :                 return 0;
    5021                 :            : 
    5022                 :          0 :         vdev = xhci->devs[hdev->slot_id];
    5023         [ #  # ]:          0 :         if (!vdev) {
    5024                 :          0 :                 xhci_warn(xhci, "Cannot update hub desc for unknown device.\n");
    5025                 :          0 :                 return -EINVAL;
    5026                 :            :         }
    5027                 :            : 
    5028                 :          0 :         config_cmd = xhci_alloc_command_with_ctx(xhci, true, mem_flags);
    5029         [ #  # ]:          0 :         if (!config_cmd)
    5030                 :            :                 return -ENOMEM;
    5031                 :            : 
    5032                 :          0 :         ctrl_ctx = xhci_get_input_control_ctx(config_cmd->in_ctx);
    5033         [ #  # ]:          0 :         if (!ctrl_ctx) {
    5034                 :          0 :                 xhci_warn(xhci, "%s: Could not get input context, bad type.\n",
    5035                 :            :                                 __func__);
    5036                 :          0 :                 xhci_free_command(xhci, config_cmd);
    5037                 :          0 :                 return -ENOMEM;
    5038                 :            :         }
    5039                 :            : 
    5040                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    5041   [ #  #  #  # ]:          0 :         if (hdev->speed == USB_SPEED_HIGH &&
    5042                 :          0 :                         xhci_alloc_tt_info(xhci, vdev, hdev, tt, GFP_ATOMIC)) {
    5043                 :          0 :                 xhci_dbg(xhci, "Could not allocate xHCI TT structure.\n");
    5044                 :          0 :                 xhci_free_command(xhci, config_cmd);
    5045                 :          0 :                 spin_unlock_irqrestore(&xhci->lock, flags);
    5046                 :          0 :                 return -ENOMEM;
    5047                 :            :         }
    5048                 :            : 
    5049                 :          0 :         xhci_slot_copy(xhci, config_cmd->in_ctx, vdev->out_ctx);
    5050                 :          0 :         ctrl_ctx->add_flags |= cpu_to_le32(SLOT_FLAG);
    5051                 :          0 :         slot_ctx = xhci_get_slot_ctx(xhci, config_cmd->in_ctx);
    5052                 :          0 :         slot_ctx->dev_info |= cpu_to_le32(DEV_HUB);
    5053                 :            :         /*
    5054                 :            :          * refer to section 6.2.2: MTT should be 0 for full speed hub,
    5055                 :            :          * but it may be already set to 1 when setup an xHCI virtual
    5056                 :            :          * device, so clear it anyway.
    5057                 :            :          */
    5058         [ #  # ]:          0 :         if (tt->multi)
    5059                 :          0 :                 slot_ctx->dev_info |= cpu_to_le32(DEV_MTT);
    5060         [ #  # ]:          0 :         else if (hdev->speed == USB_SPEED_FULL)
    5061                 :          0 :                 slot_ctx->dev_info &= cpu_to_le32(~DEV_MTT);
    5062                 :            : 
    5063         [ #  # ]:          0 :         if (xhci->hci_version > 0x95) {
    5064                 :          0 :                 xhci_dbg(xhci, "xHCI version %x needs hub "
    5065                 :            :                                 "TT think time and number of ports\n",
    5066                 :            :                                 (unsigned int) xhci->hci_version);
    5067                 :          0 :                 slot_ctx->dev_info2 |= cpu_to_le32(XHCI_MAX_PORTS(hdev->maxchild));
    5068                 :            :                 /* Set TT think time - convert from ns to FS bit times.
    5069                 :            :                  * 0 = 8 FS bit times, 1 = 16 FS bit times,
    5070                 :            :                  * 2 = 24 FS bit times, 3 = 32 FS bit times.
    5071                 :            :                  *
    5072                 :            :                  * xHCI 1.0: this field shall be 0 if the device is not a
    5073                 :            :                  * High-spped hub.
    5074                 :            :                  */
    5075                 :          0 :                 think_time = tt->think_time;
    5076         [ #  # ]:          0 :                 if (think_time != 0)
    5077                 :          0 :                         think_time = (think_time / 666) - 1;
    5078   [ #  #  #  # ]:          0 :                 if (xhci->hci_version < 0x100 || hdev->speed == USB_SPEED_HIGH)
    5079                 :          0 :                         slot_ctx->tt_info |=
    5080                 :          0 :                                 cpu_to_le32(TT_THINK_TIME(think_time));
    5081                 :            :         } else {
    5082                 :            :                 xhci_dbg(xhci, "xHCI version %x doesn't need hub "
    5083                 :            :                                 "TT think time or number of ports\n",
    5084                 :            :                                 (unsigned int) xhci->hci_version);
    5085                 :            :         }
    5086                 :          0 :         slot_ctx->dev_state = 0;
    5087                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    5088                 :            : 
    5089                 :          0 :         xhci_dbg(xhci, "Set up %s for hub device.\n",
    5090                 :            :                         (xhci->hci_version > 0x95) ?
    5091                 :            :                         "configure endpoint" : "evaluate context");
    5092                 :            : 
    5093                 :            :         /* Issue and wait for the configure endpoint or
    5094                 :            :          * evaluate context command.
    5095                 :            :          */
    5096         [ #  # ]:          0 :         if (xhci->hci_version > 0x95)
    5097                 :          0 :                 ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
    5098                 :            :                                 false, false);
    5099                 :            :         else
    5100                 :          0 :                 ret = xhci_configure_endpoint(xhci, hdev, config_cmd,
    5101                 :            :                                 true, false);
    5102                 :            : 
    5103                 :          0 :         xhci_free_command(xhci, config_cmd);
    5104                 :          0 :         return ret;
    5105                 :            : }
    5106                 :            : 
    5107                 :          0 : static int xhci_get_frame(struct usb_hcd *hcd)
    5108                 :            : {
    5109                 :          0 :         struct xhci_hcd *xhci = hcd_to_xhci(hcd);
    5110                 :            :         /* EHCI mods by the periodic size.  Why? */
    5111                 :          0 :         return readl(&xhci->run_regs->microframe_index) >> 3;
    5112                 :            : }
    5113                 :            : 
    5114                 :          0 : int xhci_gen_setup(struct usb_hcd *hcd, xhci_get_quirks_t get_quirks)
    5115                 :            : {
    5116                 :          0 :         struct xhci_hcd         *xhci;
    5117                 :            :         /*
    5118                 :            :          * TODO: Check with DWC3 clients for sysdev according to
    5119                 :            :          * quirks
    5120                 :            :          */
    5121                 :          0 :         struct device           *dev = hcd->self.sysdev;
    5122                 :          0 :         unsigned int            minor_rev;
    5123                 :          0 :         int                     retval;
    5124                 :            : 
    5125                 :            :         /* Accept arbitrarily long scatter-gather lists */
    5126                 :          0 :         hcd->self.sg_tablesize = ~0;
    5127                 :            : 
    5128                 :            :         /* support to build packet from discontinuous buffers */
    5129                 :          0 :         hcd->self.no_sg_constraint = 1;
    5130                 :            : 
    5131                 :            :         /* XHCI controllers don't stop the ep queue on short packets :| */
    5132                 :          0 :         hcd->self.no_stop_on_short = 1;
    5133                 :            : 
    5134                 :          0 :         xhci = hcd_to_xhci(hcd);
    5135                 :            : 
    5136         [ #  # ]:          0 :         if (usb_hcd_is_primary_hcd(hcd)) {
    5137                 :          0 :                 xhci->main_hcd = hcd;
    5138                 :          0 :                 xhci->usb2_rhub.hcd = hcd;
    5139                 :            :                 /* Mark the first roothub as being USB 2.0.
    5140                 :            :                  * The xHCI driver will register the USB 3.0 roothub.
    5141                 :            :                  */
    5142                 :          0 :                 hcd->speed = HCD_USB2;
    5143                 :          0 :                 hcd->self.root_hub->speed = USB_SPEED_HIGH;
    5144                 :            :                 /*
    5145                 :            :                  * USB 2.0 roothub under xHCI has an integrated TT,
    5146                 :            :                  * (rate matching hub) as opposed to having an OHCI/UHCI
    5147                 :            :                  * companion controller.
    5148                 :            :                  */
    5149                 :          0 :                 hcd->has_tt = 1;
    5150                 :            :         } else {
    5151                 :            :                 /*
    5152                 :            :                  * Early xHCI 1.1 spec did not mention USB 3.1 capable hosts
    5153                 :            :                  * should return 0x31 for sbrn, or that the minor revision
    5154                 :            :                  * is a two digit BCD containig minor and sub-minor numbers.
    5155                 :            :                  * This was later clarified in xHCI 1.2.
    5156                 :            :                  *
    5157                 :            :                  * Some USB 3.1 capable hosts therefore have sbrn 0x30, and
    5158                 :            :                  * minor revision set to 0x1 instead of 0x10.
    5159                 :            :                  */
    5160         [ #  # ]:          0 :                 if (xhci->usb3_rhub.min_rev == 0x1)
    5161                 :            :                         minor_rev = 1;
    5162                 :            :                 else
    5163                 :          0 :                         minor_rev = xhci->usb3_rhub.min_rev / 0x10;
    5164                 :            : 
    5165      [ #  #  # ]:          0 :                 switch (minor_rev) {
    5166                 :          0 :                 case 2:
    5167                 :          0 :                         hcd->speed = HCD_USB32;
    5168                 :          0 :                         hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
    5169                 :          0 :                         hcd->self.root_hub->rx_lanes = 2;
    5170                 :          0 :                         hcd->self.root_hub->tx_lanes = 2;
    5171                 :          0 :                         break;
    5172                 :          0 :                 case 1:
    5173                 :          0 :                         hcd->speed = HCD_USB31;
    5174                 :          0 :                         hcd->self.root_hub->speed = USB_SPEED_SUPER_PLUS;
    5175                 :          0 :                         break;
    5176                 :            :                 }
    5177         [ #  # ]:          0 :                 xhci_info(xhci, "Host supports USB 3.%x %sSuperSpeed\n",
    5178                 :            :                           minor_rev,
    5179                 :            :                           minor_rev ? "Enhanced " : "");
    5180                 :            : 
    5181                 :          0 :                 xhci->usb3_rhub.hcd = hcd;
    5182                 :            :                 /* xHCI private pointer was set in xhci_pci_probe for the second
    5183                 :            :                  * registered roothub.
    5184                 :            :                  */
    5185                 :          0 :                 return 0;
    5186                 :            :         }
    5187                 :            : 
    5188                 :          0 :         mutex_init(&xhci->mutex);
    5189                 :          0 :         xhci->cap_regs = hcd->regs;
    5190                 :          0 :         xhci->op_regs = hcd->regs +
    5191                 :          0 :                 HC_LENGTH(readl(&xhci->cap_regs->hc_capbase));
    5192                 :          0 :         xhci->run_regs = hcd->regs +
    5193                 :          0 :                 (readl(&xhci->cap_regs->run_regs_off) & RTSOFF_MASK);
    5194                 :            :         /* Cache read-only capability registers */
    5195                 :          0 :         xhci->hcs_params1 = readl(&xhci->cap_regs->hcs_params1);
    5196                 :          0 :         xhci->hcs_params2 = readl(&xhci->cap_regs->hcs_params2);
    5197                 :          0 :         xhci->hcs_params3 = readl(&xhci->cap_regs->hcs_params3);
    5198                 :          0 :         xhci->hcc_params = readl(&xhci->cap_regs->hc_capbase);
    5199                 :          0 :         xhci->hci_version = HC_VERSION(xhci->hcc_params);
    5200                 :          0 :         xhci->hcc_params = readl(&xhci->cap_regs->hcc_params);
    5201         [ #  # ]:          0 :         if (xhci->hci_version > 0x100)
    5202                 :          0 :                 xhci->hcc_params2 = readl(&xhci->cap_regs->hcc_params2);
    5203                 :            : 
    5204                 :          0 :         xhci->quirks |= quirks;
    5205                 :            : 
    5206                 :          0 :         get_quirks(dev, xhci);
    5207                 :            : 
    5208                 :            :         /* In xhci controllers which follow xhci 1.0 spec gives a spurious
    5209                 :            :          * success event after a short transfer. This quirk will ignore such
    5210                 :            :          * spurious event.
    5211                 :            :          */
    5212         [ #  # ]:          0 :         if (xhci->hci_version > 0x96)
    5213                 :          0 :                 xhci->quirks |= XHCI_SPURIOUS_SUCCESS;
    5214                 :            : 
    5215                 :            :         /* Make sure the HC is halted. */
    5216                 :          0 :         retval = xhci_halt(xhci);
    5217         [ #  # ]:          0 :         if (retval)
    5218                 :            :                 return retval;
    5219                 :            : 
    5220                 :          0 :         xhci_zero_64b_regs(xhci);
    5221                 :            : 
    5222                 :          0 :         xhci_dbg(xhci, "Resetting HCD\n");
    5223                 :            :         /* Reset the internal HC memory state and registers. */
    5224                 :          0 :         retval = xhci_reset(xhci);
    5225         [ #  # ]:          0 :         if (retval)
    5226                 :            :                 return retval;
    5227                 :          0 :         xhci_dbg(xhci, "Reset complete\n");
    5228                 :            : 
    5229                 :            :         /*
    5230                 :            :          * On some xHCI controllers (e.g. R-Car SoCs), the AC64 bit (bit 0)
    5231                 :            :          * of HCCPARAMS1 is set to 1. However, the xHCs don't support 64-bit
    5232                 :            :          * address memory pointers actually. So, this driver clears the AC64
    5233                 :            :          * bit of xhci->hcc_params to call dma_set_coherent_mask(dev,
    5234                 :            :          * DMA_BIT_MASK(32)) in this xhci_gen_setup().
    5235                 :            :          */
    5236         [ #  # ]:          0 :         if (xhci->quirks & XHCI_NO_64BIT_SUPPORT)
    5237                 :          0 :                 xhci->hcc_params &= ~BIT(0);
    5238                 :            : 
    5239                 :            :         /* Set dma_mask and coherent_dma_mask to 64-bits,
    5240                 :            :          * if xHC supports 64-bit addressing */
    5241   [ #  #  #  # ]:          0 :         if (HCC_64BIT_ADDR(xhci->hcc_params) &&
    5242                 :          0 :                         !dma_set_mask(dev, DMA_BIT_MASK(64))) {
    5243                 :          0 :                 xhci_dbg(xhci, "Enabling 64-bit DMA addresses.\n");
    5244                 :          0 :                 dma_set_coherent_mask(dev, DMA_BIT_MASK(64));
    5245                 :            :         } else {
    5246                 :            :                 /*
    5247                 :            :                  * This is to avoid error in cases where a 32-bit USB
    5248                 :            :                  * controller is used on a 64-bit capable system.
    5249                 :            :                  */
    5250                 :          0 :                 retval = dma_set_mask(dev, DMA_BIT_MASK(32));
    5251         [ #  # ]:          0 :                 if (retval)
    5252                 :            :                         return retval;
    5253                 :          0 :                 xhci_dbg(xhci, "Enabling 32-bit DMA addresses.\n");
    5254                 :          0 :                 dma_set_coherent_mask(dev, DMA_BIT_MASK(32));
    5255                 :            :         }
    5256                 :            : 
    5257                 :          0 :         xhci_dbg(xhci, "Calling HCD init\n");
    5258                 :            :         /* Initialize HCD and host controller data structures. */
    5259                 :          0 :         retval = xhci_init(hcd);
    5260         [ #  # ]:          0 :         if (retval)
    5261                 :            :                 return retval;
    5262                 :          0 :         xhci_dbg(xhci, "Called HCD init\n");
    5263                 :            : 
    5264                 :          0 :         xhci_info(xhci, "hcc params 0x%08x hci version 0x%x quirks 0x%016llx\n",
    5265                 :            :                   xhci->hcc_params, xhci->hci_version, xhci->quirks);
    5266                 :            : 
    5267                 :          0 :         return 0;
    5268                 :            : }
    5269                 :            : EXPORT_SYMBOL_GPL(xhci_gen_setup);
    5270                 :            : 
    5271                 :          0 : static void xhci_clear_tt_buffer_complete(struct usb_hcd *hcd,
    5272                 :            :                 struct usb_host_endpoint *ep)
    5273                 :            : {
    5274                 :          0 :         struct xhci_hcd *xhci;
    5275                 :          0 :         struct usb_device *udev;
    5276                 :          0 :         unsigned int slot_id;
    5277                 :          0 :         unsigned int ep_index;
    5278                 :          0 :         unsigned long flags;
    5279                 :            : 
    5280                 :          0 :         xhci = hcd_to_xhci(hcd);
    5281                 :            : 
    5282                 :          0 :         spin_lock_irqsave(&xhci->lock, flags);
    5283                 :          0 :         udev = (struct usb_device *)ep->hcpriv;
    5284                 :          0 :         slot_id = udev->slot_id;
    5285         [ #  # ]:          0 :         ep_index = xhci_get_endpoint_index(&ep->desc);
    5286                 :            : 
    5287                 :          0 :         xhci->devs[slot_id]->eps[ep_index].ep_state &= ~EP_CLEARING_TT;
    5288                 :          0 :         xhci_ring_doorbell_for_active_rings(xhci, slot_id, ep_index);
    5289                 :          0 :         spin_unlock_irqrestore(&xhci->lock, flags);
    5290                 :          0 : }
    5291                 :            : 
    5292                 :            : static const struct hc_driver xhci_hc_driver = {
    5293                 :            :         .description =          "xhci-hcd",
    5294                 :            :         .product_desc =         "xHCI Host Controller",
    5295                 :            :         .hcd_priv_size =        sizeof(struct xhci_hcd),
    5296                 :            : 
    5297                 :            :         /*
    5298                 :            :          * generic hardware linkage
    5299                 :            :          */
    5300                 :            :         .irq =                  xhci_irq,
    5301                 :            :         .flags =                HCD_MEMORY | HCD_DMA | HCD_USB3 | HCD_SHARED |
    5302                 :            :                                 HCD_BH,
    5303                 :            : 
    5304                 :            :         /*
    5305                 :            :          * basic lifecycle operations
    5306                 :            :          */
    5307                 :            :         .reset =                NULL, /* set in xhci_init_driver() */
    5308                 :            :         .start =                xhci_run,
    5309                 :            :         .stop =                 xhci_stop,
    5310                 :            :         .shutdown =             xhci_shutdown,
    5311                 :            : 
    5312                 :            :         /*
    5313                 :            :          * managing i/o requests and associated device resources
    5314                 :            :          */
    5315                 :            :         .map_urb_for_dma =      xhci_map_urb_for_dma,
    5316                 :            :         .urb_enqueue =          xhci_urb_enqueue,
    5317                 :            :         .urb_dequeue =          xhci_urb_dequeue,
    5318                 :            :         .alloc_dev =            xhci_alloc_dev,
    5319                 :            :         .free_dev =             xhci_free_dev,
    5320                 :            :         .alloc_streams =        xhci_alloc_streams,
    5321                 :            :         .free_streams =         xhci_free_streams,
    5322                 :            :         .add_endpoint =         xhci_add_endpoint,
    5323                 :            :         .drop_endpoint =        xhci_drop_endpoint,
    5324                 :            :         .endpoint_disable =     xhci_endpoint_disable,
    5325                 :            :         .endpoint_reset =       xhci_endpoint_reset,
    5326                 :            :         .check_bandwidth =      xhci_check_bandwidth,
    5327                 :            :         .reset_bandwidth =      xhci_reset_bandwidth,
    5328                 :            :         .address_device =       xhci_address_device,
    5329                 :            :         .enable_device =        xhci_enable_device,
    5330                 :            :         .update_hub_device =    xhci_update_hub_device,
    5331                 :            :         .reset_device =         xhci_discover_or_reset_device,
    5332                 :            : 
    5333                 :            :         /*
    5334                 :            :          * scheduling support
    5335                 :            :          */
    5336                 :            :         .get_frame_number =     xhci_get_frame,
    5337                 :            : 
    5338                 :            :         /*
    5339                 :            :          * root hub support
    5340                 :            :          */
    5341                 :            :         .hub_control =          xhci_hub_control,
    5342                 :            :         .hub_status_data =      xhci_hub_status_data,
    5343                 :            :         .bus_suspend =          xhci_bus_suspend,
    5344                 :            :         .bus_resume =           xhci_bus_resume,
    5345                 :            :         .get_resuming_ports =   xhci_get_resuming_ports,
    5346                 :            : 
    5347                 :            :         /*
    5348                 :            :          * call back when device connected and addressed
    5349                 :            :          */
    5350                 :            :         .update_device =        xhci_update_device,
    5351                 :            :         .set_usb2_hw_lpm =      xhci_set_usb2_hardware_lpm,
    5352                 :            :         .enable_usb3_lpm_timeout =      xhci_enable_usb3_lpm_timeout,
    5353                 :            :         .disable_usb3_lpm_timeout =     xhci_disable_usb3_lpm_timeout,
    5354                 :            :         .find_raw_port_number = xhci_find_raw_port_number,
    5355                 :            :         .clear_tt_buffer_complete = xhci_clear_tt_buffer_complete,
    5356                 :            : };
    5357                 :            : 
    5358                 :         13 : void xhci_init_driver(struct hc_driver *drv,
    5359                 :            :                       const struct xhci_driver_overrides *over)
    5360                 :            : {
    5361         [ -  + ]:         13 :         BUG_ON(!over);
    5362                 :            : 
    5363                 :            :         /* Copy the generic table to drv then apply the overrides */
    5364                 :         13 :         *drv = xhci_hc_driver;
    5365                 :            : 
    5366         [ +  - ]:         13 :         if (over) {
    5367                 :         13 :                 drv->hcd_priv_size += over->extra_priv_size;
    5368         [ +  - ]:         13 :                 if (over->reset)
    5369                 :         13 :                         drv->reset = over->reset;
    5370         [ -  + ]:         13 :                 if (over->start)
    5371                 :          0 :                         drv->start = over->start;
    5372                 :            :         }
    5373                 :         13 : }
    5374                 :            : EXPORT_SYMBOL_GPL(xhci_init_driver);
    5375                 :            : 
    5376                 :            : MODULE_DESCRIPTION(DRIVER_DESC);
    5377                 :            : MODULE_AUTHOR(DRIVER_AUTHOR);
    5378                 :            : MODULE_LICENSE("GPL");
    5379                 :            : 
    5380                 :         13 : static int __init xhci_hcd_init(void)
    5381                 :            : {
    5382                 :            :         /*
    5383                 :            :          * Check the compiler generated sizes of structures that must be laid
    5384                 :            :          * out in specific ways for hardware access.
    5385                 :            :          */
    5386                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_doorbell_array) != 256*32/8);
    5387                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_slot_ctx) != 8*32/8);
    5388                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_ep_ctx) != 8*32/8);
    5389                 :            :         /* xhci_device_control has eight fields, and also
    5390                 :            :          * embeds one xhci_slot_ctx and 31 xhci_ep_ctx
    5391                 :            :          */
    5392                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_stream_ctx) != 4*32/8);
    5393                 :         13 :         BUILD_BUG_ON(sizeof(union xhci_trb) != 4*32/8);
    5394                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_erst_entry) != 4*32/8);
    5395                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_cap_regs) != 8*32/8);
    5396                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_intr_reg) != 8*32/8);
    5397                 :            :         /* xhci_run_regs has eight fields and embeds 128 xhci_intr_regs */
    5398                 :         13 :         BUILD_BUG_ON(sizeof(struct xhci_run_regs) != (8+8*128)*32/8);
    5399                 :            : 
    5400         [ +  - ]:         13 :         if (usb_disabled())
    5401                 :            :                 return -ENODEV;
    5402                 :            : 
    5403                 :         13 :         xhci_debugfs_create_root();
    5404                 :            : 
    5405                 :         13 :         return 0;
    5406                 :            : }
    5407                 :            : 
    5408                 :            : /*
    5409                 :            :  * If an init function is provided, an exit function must also be provided
    5410                 :            :  * to allow module unload.
    5411                 :            :  */
    5412                 :          0 : static void __exit xhci_hcd_fini(void)
    5413                 :            : {
    5414                 :          0 :         xhci_debugfs_remove_root();
    5415                 :          0 : }
    5416                 :            : 
    5417                 :            : module_init(xhci_hcd_init);
    5418                 :            : module_exit(xhci_hcd_fini);

Generated by: LCOV version 1.14