drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 #define DWC3_ALIGN_FRAME(d)     (((d)->frame_number + (d)->interval) \
  31                                         & ~((d)->interval - 1))
  32
  33 /**
  34  * dwc3_gadget_set_test_mode - enables usb2 test modes
  35  * @dwc: pointer to our context structure
  36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  37  *
  38  * Caller should take care of locking. This function will return 0 on
  39  * success or -EINVAL if wrong Test Selector is passed.
  40  */
  41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  42 {
  43         u32             reg;
  44
  45         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  46         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  47
  48         switch (mode) {
  49         case TEST_J:
  50         case TEST_K:
  51         case TEST_SE0_NAK:
  52         case TEST_PACKET:
  53         case TEST_FORCE_EN:
  54                 reg |= mode << 1;
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  61
  62         return 0;
  63 }
  64
  65 /**
  66  * dwc3_gadget_get_link_state - gets current state of usb link
  67  * @dwc: pointer to our context structure
  68  *
  69  * Caller should take care of locking. This function will
  70  * return the link state on success (>= 0) or -ETIMEDOUT.
  71  */
  72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  73 {
  74         u32             reg;
  75
  76         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  77
  78         return DWC3_DSTS_USBLNKST(reg);
  79 }
  80
  81 /**
  82  * dwc3_gadget_set_link_state - sets usb link to a particular state
  83  * @dwc: pointer to our context structure
  84  * @state: the state to put link into
  85  *
  86  * Caller should take care of locking. This function will
  87  * return 0 on success or -ETIMEDOUT.
  88  */
  89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  90 {
  91         int             retries = 10000;
  92         u32             reg;
  93
  94         /*
  95          * Wait until device controller is ready. Only applies to 1.94a and
  96          * later RTL.
  97          */
  98         if (dwc->revision >= DWC3_REVISION_194A) {
  99                 while (--retries) {
 100                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 101                         if (reg & DWC3_DSTS_DCNRD)
 102                                 udelay(5);
 103                         else
 104                                 break;
 105                 }
 106
 107                 if (retries <= 0)
 108                         return -ETIMEDOUT;
 109         }
 110
 111         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 112         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 113
 114         /* set requested state */
 115         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 116         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 117
 118         /*
 119          * The following code is racy when called from dwc3_gadget_wakeup,
 120          * and is not needed, at least on newer versions
 121          */
 122         if (dwc->revision >= DWC3_REVISION_194A)
 123                 return 0;
 124
 125         /* wait for a change in DSTS */
 126         retries = 10000;
 127         while (--retries) {
 128                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 129
 130                 if (DWC3_DSTS_USBLNKST(reg) == state)
 131                         return 0;
 132
 133                 udelay(5);
 134         }
 135
 136         return -ETIMEDOUT;
 137 }
 138
 139 /**
 140  * dwc3_ep_inc_trb - increment a trb index.
 141  * @index: Pointer to the TRB index to increment.
 142  *
 143  * The index should never point to the link TRB. After incrementing,
 144  * if it is point to the link TRB, wrap around to the beginning. The
 145  * link TRB is always at the last TRB entry.
 146  */
 147 static void dwc3_ep_inc_trb(u8 *index)
 148 {
 149         (*index)++;
 150         if (*index == (DWC3_TRB_NUM - 1))
 151                 *index = 0;
 152 }
 153
 154 /**
 155  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 156  * @dep: The endpoint whose enqueue pointer we're incrementing
 157  */
 158 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 159 {
 160         dwc3_ep_inc_trb(&dep->trb_enqueue);
 161 }
 162
 163 /**
 164  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 165  * @dep: The endpoint whose enqueue pointer we're incrementing
 166  */
 167 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 168 {
 169         dwc3_ep_inc_trb(&dep->trb_dequeue);
 170 }
 171
 172 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 173                 struct dwc3_request *req, int status)
 174 {
 175         struct dwc3                     *dwc = dep->dwc;
 176
 177         req->started = false;
 178         list_del(&req->list);
 179         req->remaining = 0;
 180
 181         if (req->request.status == -EINPROGRESS)
 182                 req->request.status = status;
 183
 184         if (req->trb)
 185                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 186                                 &req->request, req->direction);
 187
 188         req->trb = NULL;
 189         trace_dwc3_gadget_giveback(req);
 190
 191         if (dep->number > 1)
 192                 pm_runtime_put(dwc->dev);
 193 }
 194
 195 /**
 196  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 197  * @dep: The endpoint to whom the request belongs to
 198  * @req: The request we're giving back
 199  * @status: completion code for the request
 200  *
 201  * Must be called with controller's lock held and interrupts disabled. This
 202  * function will unmap @req and call its ->complete() callback to notify upper
 203  * layers that it has completed.
 204  */
 205 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 206                 int status)
 207 {
 208         struct dwc3                     *dwc = dep->dwc;
 209
 210         if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
 211                 del_timer(&req->stream_timeout_timer);
 212
 213         dwc3_gadget_del_and_unmap_request(dep, req, status);
 214
 215         spin_unlock(&dwc->lock);
 216         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 217         spin_lock(&dwc->lock);
 218 }
 219
 220 /**
 221  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 222  * @dwc: pointer to the controller context
 223  * @cmd: the command to be issued
 224  * @param: command parameter
 225  *
 226  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 227  * and wait for its completion.
 228  */
 229 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 230 {
 231         u32             timeout = 500;
 232         int             status = 0;
 233         int             ret = 0;
 234         u32             reg;
 235
 236         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 237         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 238
 239         do {
 240                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 241                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 242                         status = DWC3_DGCMD_STATUS(reg);
 243                         if (status)
 244                                 ret = -EINVAL;
 245                         break;
 246                 }
 247         } while (--timeout);
 248
 249         if (!timeout) {
 250                 ret = -ETIMEDOUT;
 251                 status = -ETIMEDOUT;
 252         }
 253
 254         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 255
 256         return ret;
 257 }
 258
 259 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 260
 261 /**
 262  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 263  * @dep: the endpoint to which the command is going to be issued
 264  * @cmd: the command to be issued
 265  * @params: parameters to the command
 266  *
 267  * Caller should handle locking. This function will issue @cmd with given
 268  * @params to @dep and wait for its completion.
 269  */
 270 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 271                 struct dwc3_gadget_ep_cmd_params *params)
 272 {
 273         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 274         struct dwc3             *dwc = dep->dwc;
 275         u32                     timeout = 1000;
 276         u32                     reg;
 277
 278         int                     cmd_status = 0;
 279         int                     susphy = false;
 280         int                     ret = -EINVAL;
 281
 282         /*
 283          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 284          * we're issuing an endpoint command, we must check if
 285          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 286          *
 287          * We will also set SUSPHY bit to what it was before returning as stated
 288          * by the same section on Synopsys databook.
 289          */
 290         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 291                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 292                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 293                         susphy = true;
 294                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 295                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 296                 }
 297         }
 298
 299         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 300                 int             needs_wakeup;
 301
 302                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 303                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 304                                 dwc->link_state == DWC3_LINK_STATE_U3);
 305
 306                 if (unlikely(needs_wakeup)) {
 307                         ret = __dwc3_gadget_wakeup(dwc);
 308                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 309                                         ret);
 310                 }
 311         }
 312
 313         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 314         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 315         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 316
 317         /*
 318          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 319          * not relying on XferNotReady, we can make use of a special "No
 320          * Response Update Transfer" command where we should clear both CmdAct
 321          * and CmdIOC bits.
 322          *
 323          * With this, we don't need to wait for command completion and can
 324          * straight away issue further commands to the endpoint.
 325          *
 326          * NOTICE: We're making an assumption that control endpoints will never
 327          * make use of Update Transfer command. This is a safe assumption
 328          * because we can never have more than one request at a time with
 329          * Control Endpoints. If anybody changes that assumption, this chunk
 330          * needs to be updated accordingly.
 331          */
 332         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 333                         !usb_endpoint_xfer_isoc(desc))
 334                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 335         else
 336                 cmd |= DWC3_DEPCMD_CMDACT;
 337
 338         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 339         do {
 340                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 341                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 342                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 343
 344                         switch (cmd_status) {
 345                         case 0:
 346                                 ret = 0;
 347                                 break;
 348                         case DEPEVT_TRANSFER_NO_RESOURCE:
 349                                 ret = -EINVAL;
 350                                 break;
 351                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 352                                 /*
 353                                  * SW issues START TRANSFER command to
 354                                  * isochronous ep with future frame interval. If
 355                                  * future interval time has already passed when
 356                                  * core receives the command, it will respond
 357                                  * with an error status of 'Bus Expiry'.
 358                                  *
 359                                  * Instead of always returning -EINVAL, let's
 360                                  * give a hint to the gadget driver that this is
 361                                  * the case by returning -EAGAIN.
 362                                  */
 363                                 ret = -EAGAIN;
 364                                 break;
 365                         default:
 366                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 367                         }
 368
 369                         break;
 370                 }
 371         } while (--timeout);
 372
 373         if (timeout == 0) {
 374                 ret = -ETIMEDOUT;
 375                 cmd_status = -ETIMEDOUT;
 376         }
 377
 378         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 379
 380         if (ret == 0) {
 381                 switch (DWC3_DEPCMD_CMD(cmd)) {
 382                 case DWC3_DEPCMD_STARTTRANSFER:
 383                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 384                         dwc3_gadget_ep_get_transfer_index(dep);
 385                         break;
 386                 case DWC3_DEPCMD_ENDTRANSFER:
 387                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 388                         break;
 389                 default:
 390                         /* nothing */
 391                         break;
 392                 }
 393         }
 394
 395         if (unlikely(susphy)) {
 396                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 397                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 398                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 399         }
 400
 401         return ret;
 402 }
 403
 404 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 405 {
 406         struct dwc3 *dwc = dep->dwc;
 407         struct dwc3_gadget_ep_cmd_params params;
 408         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 409
 410         /*
 411          * As of core revision 2.60a the recommended programming model
 412          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 413          * command for IN endpoints. This is to prevent an issue where
 414          * some (non-compliant) hosts may not send ACK TPs for pending
 415          * IN transfers due to a mishandled error condition. Synopsys
 416          * STAR 9000614252.
 417          */
 418         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
 419             (dwc->gadget.speed >= USB_SPEED_SUPER))
 420                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 421
 422         memset(&params, 0, sizeof(params));
 423
 424         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 425 }
 426
 427 dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, struct dwc3_trb *trb)
 428 {
 429         u32             offset = (char *) trb - (char *) dep->trb_pool;
 430
 431         return dep->trb_pool_dma + offset;
 432 }
 433
 434 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 435 {
 436         struct dwc3             *dwc = dep->dwc;
 437
 438         if (dep->trb_pool)
 439                 return 0;
 440
 441         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 442                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 443                         &dep->trb_pool_dma, GFP_KERNEL);
 444         if (!dep->trb_pool) {
 445                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 446                                 dep->name);
 447                 return -ENOMEM;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 454 {
 455         struct dwc3             *dwc = dep->dwc;
 456
 457         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 458                         dep->trb_pool, dep->trb_pool_dma);
 459
 460         dep->trb_pool = NULL;
 461         dep->trb_pool_dma = 0;
 462 }
 463
 464 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
 465 {
 466         struct dwc3_gadget_ep_cmd_params params;
 467
 468         memset(&params, 0x00, sizeof(params));
 469
 470         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 471
 472         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 473                         &params);
 474 }
 475
 476 /**
 477  * dwc3_gadget_start_config - configure ep resources
 478  * @dep: endpoint that is being enabled
 479  *
 480  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 481  * completion, it will set Transfer Resource for all available endpoints.
 482  *
 483  * The assignment of transfer resources cannot perfectly follow the data book
 484  * due to the fact that the controller driver does not have all knowledge of the
 485  * configuration in advance. It is given this information piecemeal by the
 486  * composite gadget framework after every SET_CONFIGURATION and
 487  * SET_INTERFACE. Trying to follow the databook programming model in this
 488  * scenario can cause errors. For two reasons:
 489  *
 490  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 491  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 492  * incorrect in the scenario of multiple interfaces.
 493  *
 494  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 495  * endpoint on alt setting (8.1.6).
 496  *
 497  * The following simplified method is used instead:
 498  *
 499  * All hardware endpoints can be assigned a transfer resource and this setting
 500  * will stay persistent until either a core reset or hibernation. So whenever we
 501  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 502  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 503  * guaranteed that there are as many transfer resources as endpoints.
 504  *
 505  * This function is called for each endpoint when it is being enabled but is
 506  * triggered only when called for EP0-out, which always happens first, and which
 507  * should only happen in one of the above conditions.
 508  */
 509 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 510 {
 511         struct dwc3_gadget_ep_cmd_params params;
 512         struct dwc3             *dwc;
 513         u32                     cmd;
 514         int                     i;
 515         int                     ret;
 516
 517         if (dep->number)
 518                 return 0;
 519
 520         memset(&params, 0x00, sizeof(params));
 521         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 522         dwc = dep->dwc;
 523
 524         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 525         if (ret)
 526                 return ret;
 527
 528         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 529                 struct dwc3_ep *dep = dwc->eps[i];
 530
 531                 if (!dep)
 532                         continue;
 533
 534                 ret = dwc3_gadget_set_xfer_resource(dep);
 535                 if (ret)
 536                         return ret;
 537         }
 538
 539         return 0;
 540 }
 541
 542 static void stream_timeout_function(struct timer_list *arg)
 543 {
 544         struct dwc3_request     *req = from_timer(req, arg, stream_timeout_timer);
 545         struct dwc3_ep          *dep = req->dep;
 546         struct dwc3             *dwc = dep->dwc;
 547         unsigned long           flags;
 548
 549         spin_lock_irqsave(&dwc->lock, flags);
 550         dwc3_stop_active_transfer(dep, true);
 551         __dwc3_gadget_kick_transfer(dep);
 552         spin_unlock_irqrestore(&dwc->lock, flags);
 553 }
 554
 555 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 556 {
 557         const struct usb_ss_ep_comp_descriptor *comp_desc;
 558         const struct usb_endpoint_descriptor *desc;
 559         struct dwc3_gadget_ep_cmd_params params;
 560         struct dwc3 *dwc = dep->dwc;
 561
 562         comp_desc = dep->endpoint.comp_desc;
 563         desc = dep->endpoint.desc;
 564
 565         memset(&params, 0x00, sizeof(params));
 566
 567         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 568                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 569
 570         /* Burst size is only needed in SuperSpeed mode */
 571         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 572                 u32 burst = dep->endpoint.maxburst;
 573                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 574         }
 575
 576         params.param0 |= action;
 577         if (action == DWC3_DEPCFG_ACTION_RESTORE)
 578                 params.param2 |= dep->saved_state;
 579
 580         if (usb_endpoint_xfer_control(desc))
 581                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 582
 583         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 584                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 585
 586         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 587                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 588                         | DWC3_DEPCFG_STREAM_EVENT_EN
 589                         | DWC3_DEPCFG_XFER_COMPLETE_EN;
 590                 dep->stream_capable = true;
 591         }
 592
 593         if (!usb_endpoint_xfer_control(desc))
 594                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 595
 596         /*
 597          * We are doing 1:1 mapping for endpoints, meaning
 598          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 599          * so on. We consider the direction bit as part of the physical
 600          * endpoint number. So USB endpoint 0x81 is 0x03.
 601          */
 602         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 603
 604         /*
 605          * We must use the lower 16 TX FIFOs even though
 606          * HW might have more
 607          */
 608         if (dep->direction)
 609                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 610
 611         if (desc->bInterval) {
 612                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 613                 dep->interval = 1 << (desc->bInterval - 1);
 614         }
 615
 616         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 617 }
 618
 619 /**
 620  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 621  * @dep: endpoint to be initialized
 622  * @action: one of INIT, MODIFY or RESTORE
 623  *
 624  * Caller should take care of locking. Execute all necessary commands to
 625  * initialize a HW endpoint so it can be used by a gadget driver.
 626  */
 627 int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 628 {
 629         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 630         struct dwc3             *dwc = dep->dwc;
 631
 632         u32                     reg;
 633         int                     ret;
 634
 635         if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
 636                 ret = dwc3_gadget_start_config(dep);
 637                 if (ret)
 638                         return ret;
 639         }
 640
 641         ret = dwc3_gadget_set_ep_config(dep, action);
 642         if (ret)
 643                 return ret;
 644
 645         if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
 646                 struct dwc3_trb *trb_st_hw;
 647                 struct dwc3_trb *trb_link;
 648
 649                 dep->type = usb_endpoint_type(desc);
 650                 dep->flags |= DWC3_EP_ENABLED;
 651                 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 652
 653                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 654                 reg |= DWC3_DALEPENA_EP(dep->number);
 655                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 656
 657                 init_waitqueue_head(&dep->wait_end_transfer);
 658
 659                 if (usb_endpoint_xfer_control(desc))
 660                         goto out;
 661
 662                 if (!dwc->is_hibernated) {
 663                         /* Initialize the TRB ring */
 664                         dep->trb_dequeue = 0;
 665                         dep->trb_enqueue = 0;
 666                         memset(dep->trb_pool, 0,
 667                                sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 668                 }
 669
 670                 /* Link TRB. The HWO bit is never reset */
 671                 trb_st_hw = &dep->trb_pool[0];
 672
 673                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 674                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 675                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 676                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 677                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 678         }
 679
 680         /*
 681          * Issue StartTransfer here with no-op TRB so we can always rely on No
 682          * Response Update Transfer command.
 683          */
 684         if (((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) ||
 685              usb_endpoint_xfer_int(desc)) && !dwc->is_hibernated) {
 686                 struct dwc3_gadget_ep_cmd_params params;
 687                 struct dwc3_trb *trb;
 688                 dma_addr_t trb_dma;
 689                 u32 cmd;
 690
 691                 memset(&params, 0, sizeof(params));
 692                 trb = &dep->trb_pool[0];
 693                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 694
 695                 params.param0 = upper_32_bits(trb_dma);
 696                 params.param1 = lower_32_bits(trb_dma);
 697
 698                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 699
 700                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 701                 if (ret < 0)
 702                         return ret;
 703         }
 704
 705 out:
 706         trace_dwc3_gadget_ep_enable(dep);
 707
 708         return 0;
 709 }
 710
 711 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 712 {
 713         struct dwc3_request             *req;
 714
 715         dwc3_stop_active_transfer(dep, true);
 716
 717         /* - giveback all requests to gadget driver */
 718         while (!list_empty(&dep->started_list)) {
 719                 req = next_request(&dep->started_list);
 720
 721                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 722         }
 723
 724         while (!list_empty(&dep->pending_list)) {
 725                 req = next_request(&dep->pending_list);
 726
 727                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 728         }
 729 }
 730
 731 /**
 732  * __dwc3_gadget_ep_disable - disables a hw endpoint
 733  * @dep: the endpoint to disable
 734  *
 735  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 736  * requests which are currently being processed by the hardware and those which
 737  * are not yet scheduled.
 738  *
 739  * Caller should take care of locking.
 740  */
 741 int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 742 {
 743         struct dwc3             *dwc = dep->dwc;
 744         u32                     reg;
 745
 746         trace_dwc3_gadget_ep_disable(dep);
 747
 748         dwc3_remove_requests(dwc, dep);
 749
 750         /* make sure HW endpoint isn't stalled */
 751         if (dep->flags & DWC3_EP_STALL)
 752                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 753
 754         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 755         reg &= ~DWC3_DALEPENA_EP(dep->number);
 756         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 757
 758         dep->stream_capable = false;
 759         dep->type = 0;
 760         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
 761
 762         /* Clear out the ep descriptors for non-ep0 */
 763         if (dep->number > 1) {
 764                 dep->endpoint.comp_desc = NULL;
 765                 dep->endpoint.desc = NULL;
 766         }
 767
 768         return 0;
 769 }
 770
 771 /* -------------------------------------------------------------------------- */
 772
 773 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 774                 const struct usb_endpoint_descriptor *desc)
 775 {
 776         return -EINVAL;
 777 }
 778
 779 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 780 {
 781         return -EINVAL;
 782 }
 783
 784 /* -------------------------------------------------------------------------- */
 785
 786 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 787                 const struct usb_endpoint_descriptor *desc)
 788 {
 789         struct dwc3_ep                  *dep;
 790         struct dwc3                     *dwc;
 791         unsigned long                   flags;
 792         int                             ret;
 793
 794         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 795                 pr_debug("dwc3: invalid parameters\n");
 796                 return -EINVAL;
 797         }
 798
 799         if (!desc->wMaxPacketSize) {
 800                 pr_debug("dwc3: missing wMaxPacketSize\n");
 801                 return -EINVAL;
 802         }
 803
 804         dep = to_dwc3_ep(ep);
 805         dwc = dep->dwc;
 806
 807         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 808                                         "%s is already enabled\n",
 809                                         dep->name))
 810                 return 0;
 811
 812         spin_lock_irqsave(&dwc->lock, flags);
 813         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 814         spin_unlock_irqrestore(&dwc->lock, flags);
 815
 816         return ret;
 817 }
 818
 819 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 820 {
 821         struct dwc3_ep                  *dep;
 822         struct dwc3                     *dwc;
 823         unsigned long                   flags;
 824         int                             ret;
 825
 826         if (!ep) {
 827                 pr_debug("dwc3: invalid parameters\n");
 828                 return -EINVAL;
 829         }
 830
 831         dep = to_dwc3_ep(ep);
 832         dwc = dep->dwc;
 833
 834         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 835                                         "%s is already disabled\n",
 836                                         dep->name))
 837                 return 0;
 838
 839         spin_lock_irqsave(&dwc->lock, flags);
 840         ret = __dwc3_gadget_ep_disable(dep);
 841         spin_unlock_irqrestore(&dwc->lock, flags);
 842
 843         return ret;
 844 }
 845
 846 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 847                 gfp_t gfp_flags)
 848 {
 849         struct dwc3_request             *req;
 850         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 851
 852         req = kzalloc(sizeof(*req), gfp_flags);
 853         if (!req)
 854                 return NULL;
 855
 856         req->direction  = dep->direction;
 857         req->epnum      = dep->number;
 858         req->dep        = dep;
 859
 860         trace_dwc3_alloc_request(req);
 861
 862         return &req->request;
 863 }
 864
 865 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 866                 struct usb_request *request)
 867 {
 868         struct dwc3_request             *req = to_dwc3_request(request);
 869
 870         trace_dwc3_free_request(req);
 871         kfree(req);
 872 }
 873
 874 /**
 875  * dwc3_ep_prev_trb - returns the previous TRB in the ring
 876  * @dep: The endpoint with the TRB ring
 877  * @index: The index of the current TRB in the ring
 878  *
 879  * Returns the TRB prior to the one pointed to by the index. If the
 880  * index is 0, we will wrap backwards, skip the link TRB, and return
 881  * the one just before that.
 882  */
 883 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
 884 {
 885         u8 tmp = index;
 886
 887         if (!tmp)
 888                 tmp = DWC3_TRB_NUM - 1;
 889
 890         return &dep->trb_pool[tmp - 1];
 891 }
 892
 893 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 894 {
 895         struct dwc3_trb         *tmp;
 896         u8                      trbs_left;
 897
 898         /*
 899          * If enqueue & dequeue are equal than it is either full or empty.
 900          *
 901          * One way to know for sure is if the TRB right before us has HWO bit
 902          * set or not. If it has, then we're definitely full and can't fit any
 903          * more transfers in our ring.
 904          */
 905         if (dep->trb_enqueue == dep->trb_dequeue) {
 906                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
 907                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
 908                         return 0;
 909
 910                 return DWC3_TRB_NUM - 1;
 911         }
 912
 913         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
 914         trbs_left &= (DWC3_TRB_NUM - 1);
 915
 916         if (dep->trb_dequeue < dep->trb_enqueue)
 917                 trbs_left--;
 918
 919         return trbs_left;
 920 }
 921
 922 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 923                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 924                 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
 925 {
 926         struct dwc3             *dwc = dep->dwc;
 927         struct usb_gadget       *gadget = &dwc->gadget;
 928         enum usb_device_speed   speed = gadget->speed;
 929
 930         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 931         trb->bpl = lower_32_bits(dma);
 932         trb->bph = upper_32_bits(dma);
 933
 934         switch (usb_endpoint_type(dep->endpoint.desc)) {
 935         case USB_ENDPOINT_XFER_CONTROL:
 936                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 937                 break;
 938
 939         case USB_ENDPOINT_XFER_ISOC:
 940                 if (!node) {
 941                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 942
 943                         /*
 944                          * USB Specification 2.0 Section 5.9.2 states that: "If
 945                          * there is only a single transaction in the microframe,
 946                          * only a DATA0 data packet PID is used.  If there are
 947                          * two transactions per microframe, DATA1 is used for
 948                          * the first transaction data packet and DATA0 is used
 949                          * for the second transaction data packet.  If there are
 950                          * three transactions per microframe, DATA2 is used for
 951                          * the first transaction data packet, DATA1 is used for
 952                          * the second, and DATA0 is used for the third."
 953                          *
 954                          * IOW, we should satisfy the following cases:
 955                          *
 956                          * 1) length <= maxpacket
 957                          *      - DATA0
 958                          *
 959                          * 2) maxpacket < length <= (2 * maxpacket)
 960                          *      - DATA1, DATA0
 961                          *
 962                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 963                          *      - DATA2, DATA1, DATA0
 964                          */
 965                         if (speed == USB_SPEED_HIGH) {
 966                                 struct usb_ep *ep = &dep->endpoint;
 967                                 unsigned int mult = 2;
 968                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 969
 970                                 if (length <= (2 * maxp))
 971                                         mult--;
 972
 973                                 if (length <= maxp)
 974                                         mult--;
 975
 976                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
 977                         }
 978                 } else {
 979                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 980                 }
 981
 982                 /* always enable Interrupt on Missed ISOC */
 983                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 984                 break;
 985
 986         case USB_ENDPOINT_XFER_BULK:
 987         case USB_ENDPOINT_XFER_INT:
 988                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 989                 break;
 990         default:
 991                 /*
 992                  * This is only possible with faulty memory because we
 993                  * checked it already :)
 994                  */
 995                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
 996                                 usb_endpoint_type(dep->endpoint.desc));
 997         }
 998
 999         /* always enable Continue on Short Packet */
1000         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1001                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1002
1003                 if (short_not_ok)
1004                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1005         }
1006
1007         if ((!no_interrupt && !chain) ||
1008             (dwc3_calc_trbs_left(dep) == 1))
1009                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1010
1011         if (chain)
1012                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1013         /*
1014          * To start transfer on another stream number endpoint need to relase
1015          * previously acquired transfer resource for doing that there is two
1016          * ways 1. end transfer 2. set lst bit of control trb
1017          *
1018          * by using lst bit in ctrl trb we will be able to save the time of
1019          * ending transfer hence improved performance
1020          */
1021         else if (dep->stream_capable)
1022                 trb->ctrl |= DWC3_TRB_CTRL_LST;
1023
1024         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1025                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1026
1027         trb->ctrl |= DWC3_TRB_CTRL_HWO;
1028
1029         dwc3_ep_inc_enq(dep);
1030
1031         trace_dwc3_prepare_trb(dep, trb);
1032 }
1033
1034 /**
1035  * dwc3_prepare_one_trb - setup one TRB from one request
1036  * @dep: endpoint for which this request is prepared
1037  * @req: dwc3_request pointer
1038  * @chain: should this TRB be chained to the next?
1039  * @node: only for isochronous endpoints. First TRB needs different type.
1040  */
1041 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1042                 struct dwc3_request *req, unsigned chain, unsigned node)
1043 {
1044         struct dwc3_trb         *trb;
1045         unsigned int            length;
1046         dma_addr_t              dma;
1047         unsigned                stream_id = req->request.stream_id;
1048         unsigned                short_not_ok = req->request.short_not_ok;
1049         unsigned                no_interrupt = req->request.no_interrupt;
1050
1051         if (req->request.num_sgs > 0) {
1052                 length = sg_dma_len(req->start_sg);
1053                 dma = sg_dma_address(req->start_sg);
1054         } else {
1055                 length = req->request.length;
1056                 dma = req->request.dma;
1057         }
1058
1059         trb = &dep->trb_pool[dep->trb_enqueue];
1060
1061         if (!req->trb) {
1062                 dwc3_gadget_move_started_request(req);
1063                 req->trb = trb;
1064                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1065         }
1066
1067         __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1068                         stream_id, short_not_ok, no_interrupt);
1069 }
1070
1071 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1072                 struct dwc3_request *req)
1073 {
1074         struct scatterlist *sg = req->start_sg;
1075         struct scatterlist *s;
1076         int             i;
1077
1078         unsigned int remaining = req->request.num_mapped_sgs
1079                 - req->num_queued_sgs;
1080
1081         for_each_sg(sg, s, remaining, i) {
1082                 unsigned int length = req->request.length;
1083                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1084                 unsigned int rem = length % maxp;
1085                 unsigned chain = true;
1086
1087                 if (sg_is_last(s))
1088                         chain = false;
1089
1090                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1091                         struct dwc3     *dwc = dep->dwc;
1092                         struct dwc3_trb *trb;
1093
1094                         req->unaligned = true;
1095
1096                         /* prepare normal TRB */
1097                         dwc3_prepare_one_trb(dep, req, true, i);
1098
1099                         /* Now prepare one extra TRB to align transfer size */
1100                         trb = &dep->trb_pool[dep->trb_enqueue];
1101                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1102                                         maxp - rem, false, 0,
1103                                         req->request.stream_id,
1104                                         req->request.short_not_ok,
1105                                         req->request.no_interrupt);
1106                 } else {
1107                         dwc3_prepare_one_trb(dep, req, chain, i);
1108                 }
1109
1110                 /*
1111                  * There can be a situation where all sgs in sglist are not
1112                  * queued because of insufficient trb number. To handle this
1113                  * case, update start_sg to next sg to be queued, so that
1114                  * we have free trbs we can continue queuing from where we
1115                  * previously stopped
1116                  */
1117                 if (chain)
1118                         req->start_sg = sg_next(s);
1119
1120                 req->num_queued_sgs++;
1121
1122                 if (!dwc3_calc_trbs_left(dep))
1123                         break;
1124         }
1125 }
1126
1127 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1128                 struct dwc3_request *req)
1129 {
1130         unsigned int length = req->request.length;
1131         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1132         unsigned int rem = length % maxp;
1133
1134         if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1135                 struct dwc3     *dwc = dep->dwc;
1136                 struct dwc3_trb *trb;
1137
1138                 req->unaligned = true;
1139
1140                 /* prepare normal TRB */
1141                 dwc3_prepare_one_trb(dep, req, true, 0);
1142
1143                 /* Now prepare one extra TRB to align transfer size */
1144                 trb = &dep->trb_pool[dep->trb_enqueue];
1145                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1146                                 false, 0, req->request.stream_id,
1147                                 req->request.short_not_ok,
1148                                 req->request.no_interrupt);
1149         } else if (req->request.zero && req->request.length &&
1150                    (IS_ALIGNED(req->request.length, maxp))) {
1151                 struct dwc3     *dwc = dep->dwc;
1152                 struct dwc3_trb *trb;
1153
1154                 req->zero = true;
1155
1156                 /* prepare normal TRB */
1157                 dwc3_prepare_one_trb(dep, req, true, 0);
1158
1159                 /* Now prepare one extra TRB to handle ZLP */
1160                 trb = &dep->trb_pool[dep->trb_enqueue];
1161                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1162                                 false, 0, req->request.stream_id,
1163                                 req->request.short_not_ok,
1164                                 req->request.no_interrupt);
1165         } else {
1166                 dwc3_prepare_one_trb(dep, req, false, 0);
1167         }
1168 }
1169
1170 /*
1171  * dwc3_prepare_trbs - setup TRBs from requests
1172  * @dep: endpoint for which requests are being prepared
1173  *
1174  * The function goes through the requests list and sets up TRBs for the
1175  * transfers. The function returns once there are no more TRBs available or
1176  * it runs out of requests.
1177  */
1178 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1179 {
1180         struct dwc3_request     *req, *n;
1181
1182         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1183
1184         /*
1185          * We can get in a situation where there's a request in the started list
1186          * but there weren't enough TRBs to fully kick it in the first time
1187          * around, so it has been waiting for more TRBs to be freed up.
1188          *
1189          * In that case, we should check if we have a request with pending_sgs
1190          * in the started list and prepare TRBs for that request first,
1191          * otherwise we will prepare TRBs completely out of order and that will
1192          * break things.
1193          */
1194         list_for_each_entry(req, &dep->started_list, list) {
1195                 if (req->num_pending_sgs > 0)
1196                         dwc3_prepare_one_trb_sg(dep, req);
1197
1198                 if (!dwc3_calc_trbs_left(dep))
1199                         return;
1200         }
1201
1202         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1203                 struct dwc3     *dwc = dep->dwc;
1204                 int             ret;
1205
1206                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1207                                                     dep->direction);
1208                 if (ret)
1209                         return;
1210
1211                 req->sg                 = req->request.sg;
1212                 req->start_sg           = req->sg;
1213                 req->num_queued_sgs     = 0;
1214                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1215
1216                 if (req->num_pending_sgs > 0)
1217                         dwc3_prepare_one_trb_sg(dep, req);
1218                 else
1219                         dwc3_prepare_one_trb_linear(dep, req);
1220
1221                 if (!dwc3_calc_trbs_left(dep))
1222                         return;
1223         }
1224 }
1225
1226 int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1227 {
1228         struct dwc3_gadget_ep_cmd_params params;
1229         struct dwc3_request             *req;
1230         int                             starting;
1231         int                             ret;
1232         u32                             cmd;
1233
1234         if (!dwc3_calc_trbs_left(dep))
1235                 return 0;
1236
1237         starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1238
1239         dwc3_prepare_trbs(dep);
1240         req = next_request(&dep->started_list);
1241         if (!req) {
1242                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1243                 return 0;
1244         }
1245
1246         memset(&params, 0, sizeof(params));
1247
1248         if (starting) {
1249                 params.param0 = upper_32_bits(req->trb_dma);
1250                 params.param1 = lower_32_bits(req->trb_dma);
1251                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1252
1253                 if (dep->stream_capable)
1254                         cmd = cmd | DWC3_DEPCMD_PARAM(req->request.stream_id);
1255
1256                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1257                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1258
1259         } else {
1260                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1261                         DWC3_DEPCMD_PARAM(dep->resource_index);
1262         }
1263
1264         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1265         if (ret < 0) {
1266                 /*
1267                  * FIXME we need to iterate over the list of requests
1268                  * here and stop, unmap, free and del each of the linked
1269                  * requests instead of what we do now.
1270                  */
1271                 if (req->trb)
1272                         memset(req->trb, 0, sizeof(struct dwc3_trb));
1273                 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1274                 return ret;
1275         }
1276
1277         if (starting && dep->stream_capable) {
1278                 req->stream_timeout_timer.expires = jiffies +
1279                                 msecs_to_jiffies(STREAM_TIMEOUT_MS);
1280                 mod_timer(&req->stream_timeout_timer,
1281                           req->stream_timeout_timer.expires);
1282         }
1283
1284         return 0;
1285 }
1286
1287 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1288 {
1289         u32                     reg;
1290
1291         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1292         return DWC3_DSTS_SOFFN(reg);
1293 }
1294
1295 static void __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1296 {
1297         if (list_empty(&dep->pending_list)) {
1298                 dev_info(dep->dwc->dev, "%s: ran out of requests\n",
1299                                 dep->name);
1300                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1301                 return;
1302         }
1303
1304         dep->frame_number = DWC3_ALIGN_FRAME(dep);
1305         __dwc3_gadget_kick_transfer(dep);
1306 }
1307
1308 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc);
1309 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1310 {
1311         struct dwc3             *dwc = dep->dwc;
1312
1313         if (!dep->endpoint.desc) {
1314                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1315                                 dep->name);
1316                 return -ESHUTDOWN;
1317         }
1318
1319         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1320                                 &req->request, req->dep->name))
1321                 return -EINVAL;
1322
1323         pm_runtime_get(dwc->dev);
1324
1325         req->request.actual     = 0;
1326         req->request.status     = -EINPROGRESS;
1327
1328         if (dep->stream_capable)
1329                 timer_setup(&req->stream_timeout_timer,
1330                             stream_timeout_function, 0);
1331
1332         trace_dwc3_ep_queue(req);
1333
1334         list_add_tail(&req->list, &dep->pending_list);
1335
1336         /* If core is hibernated, need to wakeup (remote wakeup) */
1337         if (dwc->is_hibernated) {
1338                 dwc->force_hiber_wake = true;
1339                 gadget_wakeup_interrupt(dwc);
1340                 dwc->force_hiber_wake = false;
1341         }
1342
1343         /*
1344          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1345          * wait for a XferNotReady event so we will know what's the current
1346          * (micro-)frame number.
1347          *
1348          * Without this trick, we are very, very likely gonna get Bus Expiry
1349          * errors which will force us issue EndTransfer command.
1350          */
1351         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1352                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1353                         if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1354                                 dwc3_stop_active_transfer(dep, true);
1355                                 dep->flags = DWC3_EP_ENABLED;
1356                         } else {
1357                                 u32 cur_uf;
1358
1359                                 cur_uf = __dwc3_gadget_get_frame(dwc);
1360                                 __dwc3_gadget_start_isoc(dep);
1361                                 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1362                         }
1363                         return 0;
1364                 }
1365
1366                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1367                         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1368                                 __dwc3_gadget_start_isoc(dep);
1369                                 return 0;
1370                         }
1371                 }
1372         }
1373
1374         return __dwc3_gadget_kick_transfer(dep);
1375 }
1376
1377 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1378         gfp_t gfp_flags)
1379 {
1380         struct dwc3_request             *req = to_dwc3_request(request);
1381         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1382         struct dwc3                     *dwc = dep->dwc;
1383
1384         unsigned long                   flags;
1385
1386         int                             ret;
1387
1388         spin_lock_irqsave(&dwc->lock, flags);
1389         ret = __dwc3_gadget_ep_queue(dep, req);
1390         spin_unlock_irqrestore(&dwc->lock, flags);
1391
1392         return ret;
1393 }
1394
1395 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1396                 struct usb_request *request)
1397 {
1398         struct dwc3_request             *req = to_dwc3_request(request);
1399         struct dwc3_request             *r = NULL;
1400
1401         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1402         struct dwc3                     *dwc = dep->dwc;
1403
1404         unsigned long                   flags;
1405         int                             ret = 0;
1406
1407         trace_dwc3_ep_dequeue(req);
1408
1409         spin_lock_irqsave(&dwc->lock, flags);
1410
1411         if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
1412                 del_timer(&req->stream_timeout_timer);
1413
1414         list_for_each_entry(r, &dep->pending_list, list) {
1415                 if (r == req)
1416                         break;
1417         }
1418
1419         if (r != req) {
1420                 list_for_each_entry(r, &dep->started_list, list) {
1421                         if (r == req)
1422                                 break;
1423                 }
1424                 if (r == req) {
1425                         /* wait until it is processed */
1426                         dwc3_stop_active_transfer(dep, true);
1427
1428                         /*
1429                          * If request was already started, this means we had to
1430                          * stop the transfer. With that we also need to ignore
1431                          * all TRBs used by the request, however TRBs can only
1432                          * be modified after completion of END_TRANSFER
1433                          * command. So what we do here is that we wait for
1434                          * END_TRANSFER completion and only after that, we jump
1435                          * over TRBs by clearing HWO and incrementing dequeue
1436                          * pointer.
1437                          *
1438                          * Note that we have 2 possible types of transfers here:
1439                          *
1440                          * i) Linear buffer request
1441                          * ii) SG-list based request
1442                          *
1443                          * SG-list based requests will have r->num_pending_sgs
1444                          * set to a valid number (> 0). Linear requests,
1445                          * normally use a single TRB.
1446                          *
1447                          * For each of these two cases, if r->unaligned flag is
1448                          * set, one extra TRB has been used to align transfer
1449                          * size to wMaxPacketSize.
1450                          *
1451                          * All of these cases need to be taken into
1452                          * consideration so we don't mess up our TRB ring
1453                          * pointers.
1454                          */
1455                         wait_event_lock_irq(dep->wait_end_transfer,
1456                                         !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1457                                         dwc->lock);
1458
1459                         if (!r->trb)
1460                                 goto out0;
1461
1462                         if (r->num_pending_sgs) {
1463                                 struct dwc3_trb *trb;
1464                                 int i = 0;
1465
1466                                 for (i = 0; i < r->num_pending_sgs; i++) {
1467                                         trb = r->trb + i;
1468                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1469                                         dwc3_ep_inc_deq(dep);
1470                                 }
1471
1472                                 if (r->unaligned || r->zero) {
1473                                         trb = r->trb + r->num_pending_sgs + 1;
1474                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1475                                         dwc3_ep_inc_deq(dep);
1476                                 }
1477                         } else {
1478                                 struct dwc3_trb *trb = r->trb;
1479
1480                                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1481                                 dwc3_ep_inc_deq(dep);
1482
1483                                 if (r->unaligned || r->zero) {
1484                                         trb = r->trb + 1;
1485                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1486                                         dwc3_ep_inc_deq(dep);
1487                                 }
1488                         }
1489                         goto out1;
1490                 }
1491                 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1492                                 request, ep->name);
1493                 ret = -EINVAL;
1494                 goto out0;
1495         }
1496
1497 out1:
1498         /* giveback the request */
1499
1500         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1501
1502 out0:
1503         spin_unlock_irqrestore(&dwc->lock, flags);
1504
1505         return ret;
1506 }
1507
1508 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1509 {
1510         struct dwc3_gadget_ep_cmd_params        params;
1511         struct dwc3                             *dwc = dep->dwc;
1512         int                                     ret;
1513
1514         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1515                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1516                 return -EINVAL;
1517         }
1518
1519         memset(&params, 0x00, sizeof(params));
1520
1521         if (value) {
1522                 struct dwc3_trb *trb;
1523
1524                 unsigned transfer_in_flight;
1525                 unsigned started;
1526
1527                 if (dep->flags & DWC3_EP_STALL)
1528                         return 0;
1529
1530                 if (dep->number > 1)
1531                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1532                 else
1533                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1534
1535                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1536                 started = !list_empty(&dep->started_list);
1537
1538                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1539                                 (!dep->direction && started))) {
1540                         return -EAGAIN;
1541                 }
1542
1543                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1544                                 &params);
1545                 if (ret)
1546                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1547                                         dep->name);
1548                 else
1549                         dep->flags |= DWC3_EP_STALL;
1550         } else {
1551                 if (!(dep->flags & DWC3_EP_STALL))
1552                         return 0;
1553
1554                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1555                 if (ret)
1556                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1557                                         dep->name);
1558                 else
1559                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1560         }
1561
1562         return ret;
1563 }
1564
1565 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1566 {
1567         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1568         struct dwc3                     *dwc = dep->dwc;
1569
1570         unsigned long                   flags;
1571
1572         int                             ret;
1573
1574         spin_lock_irqsave(&dwc->lock, flags);
1575         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1576         spin_unlock_irqrestore(&dwc->lock, flags);
1577
1578         return ret;
1579 }
1580
1581 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1582 {
1583         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1584         struct dwc3                     *dwc = dep->dwc;
1585         unsigned long                   flags;
1586         int                             ret;
1587
1588         spin_lock_irqsave(&dwc->lock, flags);
1589         dep->flags |= DWC3_EP_WEDGE;
1590
1591         if (dep->number == 0 || dep->number == 1)
1592                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1593         else
1594                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1595         spin_unlock_irqrestore(&dwc->lock, flags);
1596
1597         return ret;
1598 }
1599
1600 /* -------------------------------------------------------------------------- */
1601
1602 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1603         .bLength        = USB_DT_ENDPOINT_SIZE,
1604         .bDescriptorType = USB_DT_ENDPOINT,
1605         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1606 };
1607
1608 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1609         .enable         = dwc3_gadget_ep0_enable,
1610         .disable        = dwc3_gadget_ep0_disable,
1611         .alloc_request  = dwc3_gadget_ep_alloc_request,
1612         .free_request   = dwc3_gadget_ep_free_request,
1613         .queue          = dwc3_gadget_ep0_queue,
1614         .dequeue        = dwc3_gadget_ep_dequeue,
1615         .set_halt       = dwc3_gadget_ep0_set_halt,
1616         .set_wedge      = dwc3_gadget_ep_set_wedge,
1617 };
1618
1619 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1620         .enable         = dwc3_gadget_ep_enable,
1621         .disable        = dwc3_gadget_ep_disable,
1622         .alloc_request  = dwc3_gadget_ep_alloc_request,
1623         .free_request   = dwc3_gadget_ep_free_request,
1624         .queue          = dwc3_gadget_ep_queue,
1625         .dequeue        = dwc3_gadget_ep_dequeue,
1626         .set_halt       = dwc3_gadget_ep_set_halt,
1627         .set_wedge      = dwc3_gadget_ep_set_wedge,
1628 };
1629
1630 /* -------------------------------------------------------------------------- */
1631
1632 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1633 {
1634         struct dwc3             *dwc = gadget_to_dwc(g);
1635
1636         return __dwc3_gadget_get_frame(dwc);
1637 }
1638
1639 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1640 {
1641         int                     retries;
1642
1643         int                     ret;
1644         u32                     reg;
1645
1646         u8                      link_state;
1647         u8                      speed;
1648
1649         /*
1650          * According to the Databook Remote wakeup request should
1651          * be issued only when the device is in early suspend state.
1652          *
1653          * We can check that via USB Link State bits in DSTS register.
1654          */
1655         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1656
1657         speed = reg & DWC3_DSTS_CONNECTSPD;
1658         if ((speed == DWC3_DSTS_SUPERSPEED) ||
1659             (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1660                 return 0;
1661
1662         link_state = DWC3_DSTS_USBLNKST(reg);
1663
1664         switch (link_state) {
1665         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1666         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1667                 break;
1668         default:
1669                 return -EINVAL;
1670         }
1671
1672         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1673         if (ret < 0) {
1674                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1675                 return ret;
1676         }
1677
1678         /* Recent versions do this automatically */
1679         if (dwc->revision < DWC3_REVISION_194A) {
1680                 /* write zeroes to Link Change Request */
1681                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1682                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1683                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1684         }
1685
1686         /* poll until Link State changes to ON */
1687         retries = 20000;
1688
1689         while (retries--) {
1690                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1691
1692                 /* in HS, means ON */
1693                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1694                         break;
1695         }
1696
1697         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1698                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1699                 return -EINVAL;
1700         }
1701
1702         return 0;
1703 }
1704
1705 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1706 {
1707         struct dwc3             *dwc = gadget_to_dwc(g);
1708         unsigned long           flags;
1709         int                     ret;
1710
1711         spin_lock_irqsave(&dwc->lock, flags);
1712         ret = __dwc3_gadget_wakeup(dwc);
1713         spin_unlock_irqrestore(&dwc->lock, flags);
1714
1715         return ret;
1716 }
1717
1718 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1719                 int is_selfpowered)
1720 {
1721         struct dwc3             *dwc = gadget_to_dwc(g);
1722         unsigned long           flags;
1723
1724         spin_lock_irqsave(&dwc->lock, flags);
1725         g->is_selfpowered = !!is_selfpowered;
1726         spin_unlock_irqrestore(&dwc->lock, flags);
1727
1728         return 0;
1729 }
1730
1731 int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1732 {
1733         u32                     reg;
1734         u32                     timeout = 500;
1735
1736         if (pm_runtime_suspended(dwc->dev))
1737                 return 0;
1738
1739         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1740         if (is_on) {
1741                 if (dwc->revision <= DWC3_REVISION_187A) {
1742                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1743                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1744                 }
1745
1746                 if (dwc->revision >= DWC3_REVISION_194A)
1747                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1748                 reg |= DWC3_DCTL_RUN_STOP;
1749
1750                 if (dwc->has_hibernation)
1751                         reg |= DWC3_DCTL_KEEP_CONNECT;
1752
1753                 dwc->pullups_connected = true;
1754         } else {
1755                 reg &= ~DWC3_DCTL_RUN_STOP;
1756
1757                 if (dwc->has_hibernation && !suspend)
1758                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1759
1760                 dwc->pullups_connected = false;
1761         }
1762
1763         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1764
1765         do {
1766                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1767                 reg &= DWC3_DSTS_DEVCTRLHLT;
1768         } while (--timeout && !(!is_on ^ !reg));
1769
1770         if (!timeout)
1771                 return -ETIMEDOUT;
1772
1773         return 0;
1774 }
1775
1776 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1777 {
1778         struct dwc3             *dwc = gadget_to_dwc(g);
1779         unsigned long           flags;
1780         int                     ret;
1781
1782         is_on = !!is_on;
1783
1784         /*
1785          * Per databook, when we want to stop the gadget, if a control transfer
1786          * is still in process, complete it and get the core into setup phase.
1787          */
1788         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1789                 reinit_completion(&dwc->ep0_in_setup);
1790
1791                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1792                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1793                 if (ret == 0) {
1794                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1795                         return -ETIMEDOUT;
1796                 }
1797         }
1798
1799         spin_lock_irqsave(&dwc->lock, flags);
1800         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1801         spin_unlock_irqrestore(&dwc->lock, flags);
1802
1803         return ret;
1804 }
1805
1806 void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1807 {
1808         u32                     reg;
1809
1810         /* Enable all but Start and End of Frame IRQs */
1811         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1812                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1813                         DWC3_DEVTEN_CMDCMPLTEN |
1814                         DWC3_DEVTEN_ERRTICERREN |
1815                         DWC3_DEVTEN_WKUPEVTEN |
1816                         DWC3_DEVTEN_CONNECTDONEEN |
1817                         DWC3_DEVTEN_USBRSTEN |
1818                         DWC3_DEVTEN_DISCONNEVTEN);
1819
1820         /* Enable hibernation IRQ */
1821         if (dwc->has_hibernation)
1822                 reg |= DWC3_DEVTEN_HIBERNATIONREQEVTEN;
1823
1824         if (dwc->revision < DWC3_REVISION_250A)
1825                 reg |= DWC3_DEVTEN_ULSTCNGEN;
1826
1827         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1828 }
1829
1830 void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1831 {
1832         /* mask all interrupts */
1833         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1834 }
1835
1836 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1837 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1838
1839 /**
1840  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1841  * @dwc: pointer to our context structure
1842  *
1843  * The following looks like complex but it's actually very simple. In order to
1844  * calculate the number of packets we can burst at once on OUT transfers, we're
1845  * gonna use RxFIFO size.
1846  *
1847  * To calculate RxFIFO size we need two numbers:
1848  * MDWIDTH = size, in bits, of the internal memory bus
1849  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1850  *
1851  * Given these two numbers, the formula is simple:
1852  *
1853  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1854  *
1855  * 24 bytes is for 3x SETUP packets
1856  * 16 bytes is a clock domain crossing tolerance
1857  *
1858  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1859  */
1860 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1861 {
1862         u32 ram2_depth;
1863         u32 mdwidth;
1864         u32 nump;
1865         u32 reg;
1866
1867         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1868         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1869
1870         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1871         nump = min_t(u32, nump, 16);
1872
1873         /* update NumP */
1874         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1875         reg &= ~DWC3_DCFG_NUMP_MASK;
1876         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1877         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1878 }
1879
1880 static int __dwc3_gadget_start(struct dwc3 *dwc)
1881 {
1882         struct dwc3_ep          *dep;
1883         int                     ret = 0;
1884         u32                     reg;
1885
1886         /*
1887          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1888          * the core supports IMOD, disable it.
1889          */
1890         if (dwc->imod_interval) {
1891                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1892                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1893         } else if (dwc3_has_imod(dwc)) {
1894                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1895         }
1896
1897         /*
1898          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1899          * field instead of letting dwc3 itself calculate that automatically.
1900          *
1901          * This way, we maximize the chances that we'll be able to get several
1902          * bursts of data without going through any sort of endpoint throttling.
1903          */
1904         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1905         if (dwc3_is_usb31(dwc))
1906                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1907         else
1908                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1909
1910         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1911
1912         dwc3_gadget_setup_nump(dwc);
1913
1914         /* For OTG mode, check if the core is currently in Host mode.
1915          * This is not an error condition as there are times when the core is
1916          * working as host and kernel is told to initiate bind operation with
1917          * gadget class driver module.
1918          * The below remaining operations are handled in OTG driver whenever
1919          * required.
1920          */
1921         if (dwc3_readl(dwc->regs, DWC3_GSTS) & DWC3_GSTS_CUR_MODE)
1922                 return 0;
1923
1924         /* Start with SuperSpeed Default */
1925         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1926
1927         dep = dwc->eps[0];
1928         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1929         if (ret) {
1930                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1931                 goto err0;
1932         }
1933
1934         dep = dwc->eps[1];
1935         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1936         if (ret) {
1937                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1938                 goto err1;
1939         }
1940
1941         /* begin to receive SETUP packets */
1942         dwc->ep0state = EP0_SETUP_PHASE;
1943         dwc3_ep0_out_start(dwc);
1944
1945         dwc3_gadget_enable_irq(dwc);
1946
1947         return 0;
1948
1949 err1:
1950         __dwc3_gadget_ep_disable(dwc->eps[0]);
1951
1952 err0:
1953         return ret;
1954 }
1955
1956 static irqreturn_t wakeup_interrupt(int irq, void *_dwc);
1957 static int dwc3_gadget_start(struct usb_gadget *g,
1958                 struct usb_gadget_driver *driver)
1959 {
1960         struct dwc3             *dwc = gadget_to_dwc(g);
1961         unsigned long           flags;
1962         int                     ret = 0;
1963         int                     irq;
1964
1965         irq = dwc->irq_gadget;
1966         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1967                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1968         if (ret) {
1969                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1970                                 irq, ret);
1971                 goto err0;
1972         }
1973
1974         /* look for wakeup interrupt if hibernation is supported */
1975         if (dwc->has_hibernation) {
1976                 irq = dwc->irq_wakeup;
1977                 ret = devm_request_irq(dwc->dev, irq, wakeup_interrupt,
1978                                        IRQF_SHARED, "usb-wakeup", dwc);
1979                 if (ret) {
1980                         dev_err(dwc->dev, "failed to request wakeup irq #%d --> %d\n",
1981                                 irq, ret);
1982                         goto err0;
1983                 }
1984         }
1985
1986         spin_lock_irqsave(&dwc->lock, flags);
1987         if (dwc->gadget_driver) {
1988                 dev_err(dwc->dev, "%s is already bound to %s\n",
1989                                 dwc->gadget.name,
1990                                 dwc->gadget_driver->driver.name);
1991                 ret = -EBUSY;
1992                 goto err1;
1993         }
1994
1995         dwc->gadget_driver      = driver;
1996
1997         if (pm_runtime_active(dwc->dev))
1998                 __dwc3_gadget_start(dwc);
1999
2000         spin_unlock_irqrestore(&dwc->lock, flags);
2001
2002         return 0;
2003
2004 err1:
2005         spin_unlock_irqrestore(&dwc->lock, flags);
2006         if (dwc->irq_gadget)
2007                 free_irq(dwc->irq_gadget, dwc->ev_buf);
2008         if (dwc->irq_wakeup)
2009                 free_irq(dwc->irq_wakeup, dwc);
2010
2011 err0:
2012         return ret;
2013 }
2014
2015 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2016 {
2017         dwc3_gadget_disable_irq(dwc);
2018         __dwc3_gadget_ep_disable(dwc->eps[0]);
2019         __dwc3_gadget_ep_disable(dwc->eps[1]);
2020 }
2021
2022 static int dwc3_gadget_stop(struct usb_gadget *g)
2023 {
2024         struct dwc3             *dwc = gadget_to_dwc(g);
2025         unsigned long           flags;
2026         int                     epnum;
2027         u32                     tmo_eps = 0;
2028
2029         spin_lock_irqsave(&dwc->lock, flags);
2030
2031         if (pm_runtime_suspended(dwc->dev))
2032                 goto out;
2033
2034         __dwc3_gadget_stop(dwc);
2035
2036         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2037                 struct dwc3_ep  *dep = dwc->eps[epnum];
2038                 int ret;
2039
2040                 if (!dep)
2041                         continue;
2042
2043                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2044                         continue;
2045
2046                 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer,
2047                             !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
2048                             dwc->lock, msecs_to_jiffies(5));
2049
2050                 if (ret <= 0) {
2051                         /* Timed out or interrupted! There's nothing much
2052                          * we can do so we just log here and print which
2053                          * endpoints timed out at the end.
2054                          */
2055                         tmo_eps |= 1 << epnum;
2056                         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
2057                 }
2058         }
2059
2060         if (tmo_eps) {
2061                 dev_err(dwc->dev,
2062                         "end transfer timed out on endpoints 0x%x [bitmap]\n",
2063                         tmo_eps);
2064         }
2065
2066 out:
2067         dwc->gadget_driver      = NULL;
2068         spin_unlock_irqrestore(&dwc->lock, flags);
2069
2070         free_irq(dwc->irq_gadget, dwc->ev_buf);
2071         free_irq(dwc->irq_wakeup, dwc);
2072
2073         return 0;
2074 }
2075
2076 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2077                                   enum usb_device_speed speed)
2078 {
2079         struct dwc3             *dwc = gadget_to_dwc(g);
2080         unsigned long           flags;
2081         u32                     reg;
2082
2083         spin_lock_irqsave(&dwc->lock, flags);
2084         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2085         reg &= ~(DWC3_DCFG_SPEED_MASK);
2086
2087         /*
2088          * WORKAROUND: DWC3 revision < 2.20a have an issue
2089          * which would cause metastability state on Run/Stop
2090          * bit if we try to force the IP to USB2-only mode.
2091          *
2092          * Because of that, we cannot configure the IP to any
2093          * speed other than the SuperSpeed
2094          *
2095          * Refers to:
2096          *
2097          * STAR#9000525659: Clock Domain Crossing on DCTL in
2098          * USB 2.0 Mode
2099          */
2100         if (dwc->revision < DWC3_REVISION_220A &&
2101             !dwc->dis_metastability_quirk) {
2102                 reg |= DWC3_DCFG_SUPERSPEED;
2103         } else {
2104                 switch (speed) {
2105                 case USB_SPEED_LOW:
2106                         reg |= DWC3_DCFG_LOWSPEED;
2107                         break;
2108                 case USB_SPEED_FULL:
2109                         reg |= DWC3_DCFG_FULLSPEED;
2110                         break;
2111                 case USB_SPEED_HIGH:
2112                         reg |= DWC3_DCFG_HIGHSPEED;
2113                         break;
2114                 case USB_SPEED_SUPER:
2115                         reg |= DWC3_DCFG_SUPERSPEED;
2116                         break;
2117                 case USB_SPEED_SUPER_PLUS:
2118                         if (dwc3_is_usb31(dwc))
2119                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2120                         else
2121                                 reg |= DWC3_DCFG_SUPERSPEED;
2122                         break;
2123                 default:
2124                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2125
2126                         if (dwc->revision & DWC3_REVISION_IS_DWC31)
2127                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2128                         else
2129                                 reg |= DWC3_DCFG_SUPERSPEED;
2130                 }
2131         }
2132         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2133
2134         spin_unlock_irqrestore(&dwc->lock, flags);
2135 }
2136
2137 static const struct usb_gadget_ops dwc3_gadget_ops = {
2138         .get_frame              = dwc3_gadget_get_frame,
2139         .wakeup                 = dwc3_gadget_wakeup,
2140         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2141         .pullup                 = dwc3_gadget_pullup,
2142         .udc_start              = dwc3_gadget_start,
2143         .udc_stop               = dwc3_gadget_stop,
2144         .udc_set_speed          = dwc3_gadget_set_speed,
2145 };
2146
2147 /* -------------------------------------------------------------------------- */
2148
2149 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2150 {
2151         struct dwc3 *dwc = dep->dwc;
2152
2153         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2154         dep->endpoint.maxburst = 1;
2155         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2156         if (!dep->direction)
2157                 dwc->gadget.ep0 = &dep->endpoint;
2158
2159         dep->endpoint.caps.type_control = true;
2160
2161         return 0;
2162 }
2163
2164 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2165 {
2166         struct dwc3 *dwc = dep->dwc;
2167         int mdwidth;
2168         int kbytes;
2169         int size;
2170
2171         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2172         /* MDWIDTH is represented in bits, we need it in bytes */
2173         mdwidth /= 8;
2174
2175         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2176         if (dwc3_is_usb31(dwc))
2177                 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2178         else
2179                 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2180
2181         /* FIFO Depth is in MDWDITH bytes. Multiply */
2182         size *= mdwidth;
2183
2184         kbytes = size / 1024;
2185         if (kbytes == 0)
2186                 kbytes = 1;
2187
2188         /*
2189          * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
2190          * internal overhead. We don't really know how these are used,
2191          * but documentation say it exists.
2192          */
2193         size -= mdwidth * (kbytes + 1);
2194         size /= kbytes;
2195
2196         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2197
2198         dep->endpoint.max_streams = 15;
2199         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2200         list_add_tail(&dep->endpoint.ep_list,
2201                         &dwc->gadget.ep_list);
2202         dep->endpoint.caps.type_iso = true;
2203         dep->endpoint.caps.type_bulk = true;
2204         dep->endpoint.caps.type_int = true;
2205
2206         return dwc3_alloc_trb_pool(dep);
2207 }
2208
2209 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2210 {
2211         struct dwc3 *dwc = dep->dwc;
2212
2213         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2214         dep->endpoint.max_streams = 15;
2215         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2216         list_add_tail(&dep->endpoint.ep_list,
2217                         &dwc->gadget.ep_list);
2218         dep->endpoint.caps.type_iso = true;
2219         dep->endpoint.caps.type_bulk = true;
2220         dep->endpoint.caps.type_int = true;
2221
2222         return dwc3_alloc_trb_pool(dep);
2223 }
2224
2225 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2226 {
2227         struct dwc3_ep                  *dep;
2228         bool                            direction = epnum & 1;
2229         int                             ret;
2230         u8                              num = epnum >> 1;
2231
2232         dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2233         if (!dep)
2234                 return -ENOMEM;
2235
2236         dep->dwc = dwc;
2237         dep->number = epnum;
2238         dep->direction = direction;
2239         dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2240         dwc->eps[epnum] = dep;
2241
2242         snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2243                         direction ? "in" : "out");
2244
2245         dep->endpoint.name = dep->name;
2246
2247         if (!(dep->number > 1)) {
2248                 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2249                 dep->endpoint.comp_desc = NULL;
2250         }
2251
2252         spin_lock_init(&dep->lock);
2253
2254         if (num == 0)
2255                 ret = dwc3_gadget_init_control_endpoint(dep);
2256         else if (direction)
2257                 ret = dwc3_gadget_init_in_endpoint(dep);
2258         else
2259                 ret = dwc3_gadget_init_out_endpoint(dep);
2260
2261         if (ret)
2262                 return ret;
2263
2264         dep->endpoint.caps.dir_in = direction;
2265         dep->endpoint.caps.dir_out = !direction;
2266
2267         INIT_LIST_HEAD(&dep->pending_list);
2268         INIT_LIST_HEAD(&dep->started_list);
2269
2270         return 0;
2271 }
2272
2273 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2274 {
2275         u8                              epnum;
2276
2277         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2278
2279         for (epnum = 0; epnum < total; epnum++) {
2280                 int                     ret;
2281
2282                 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2283                 if (ret)
2284                         return ret;
2285         }
2286
2287         return 0;
2288 }
2289
2290 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2291 {
2292         struct dwc3_ep                  *dep;
2293         u8                              epnum;
2294
2295         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2296                 dep = dwc->eps[epnum];
2297                 if (!dep)
2298                         continue;
2299                 /*
2300                  * Physical endpoints 0 and 1 are special; they form the
2301                  * bi-directional USB endpoint 0.
2302                  *
2303                  * For those two physical endpoints, we don't allocate a TRB
2304                  * pool nor do we add them the endpoints list. Due to that, we
2305                  * shouldn't do these two operations otherwise we would end up
2306                  * with all sorts of bugs when removing dwc3.ko.
2307                  */
2308                 if (epnum != 0 && epnum != 1) {
2309                         dwc3_free_trb_pool(dep);
2310                         list_del(&dep->endpoint.ep_list);
2311                 }
2312
2313                 kfree(dep);
2314         }
2315 }
2316
2317 /* -------------------------------------------------------------------------- */
2318
2319 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2320                 struct dwc3_request *req, struct dwc3_trb *trb,
2321                 const struct dwc3_event_depevt *event, int status, int chain)
2322 {
2323         unsigned int            count;
2324
2325         dwc3_ep_inc_deq(dep);
2326
2327         trace_dwc3_complete_trb(dep, trb);
2328
2329         /*
2330          * If we're in the middle of series of chained TRBs and we
2331          * receive a short transfer along the way, DWC3 will skip
2332          * through all TRBs including the last TRB in the chain (the
2333          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2334          * bit and SW has to do it manually.
2335          *
2336          * We're going to do that here to avoid problems of HW trying
2337          * to use bogus TRBs for transfers.
2338          */
2339         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2340                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2341
2342         /*
2343          * If we're dealing with unaligned size OUT transfer, we will be left
2344          * with one TRB pending in the ring. We need to manually clear HWO bit
2345          * from that TRB.
2346          */
2347         if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
2348                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2349                 return 1;
2350         }
2351
2352         count = trb->size & DWC3_TRB_SIZE_MASK;
2353         req->remaining += count;
2354
2355         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2356                 return 1;
2357
2358         if (event->status & DEPEVT_STATUS_SHORT && !chain)
2359                 return 1;
2360
2361         if ((event->status & DEPEVT_STATUS_IOC) &&
2362             (trb->ctrl & DWC3_TRB_CTRL_IOC))
2363                 return 1;
2364
2365         if ((event->status & DEPEVT_STATUS_LST) &&
2366             (trb->ctrl & DWC3_TRB_CTRL_LST))
2367                 return 1;
2368
2369         return 0;
2370 }
2371
2372 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2373                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2374                 int status)
2375 {
2376         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2377         struct scatterlist *sg = req->sg;
2378         struct scatterlist *s;
2379         unsigned int pending = req->num_pending_sgs;
2380         unsigned int i;
2381         int ret = 0;
2382
2383         for_each_sg(sg, s, pending, i) {
2384                 trb = &dep->trb_pool[dep->trb_dequeue];
2385
2386                 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2387                         break;
2388
2389                 req->sg = sg_next(s);
2390                 req->num_pending_sgs--;
2391
2392                 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2393                                 trb, event, status, true);
2394                 if (ret)
2395                         break;
2396         }
2397
2398         return ret;
2399 }
2400
2401 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2402                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2403                 int status)
2404 {
2405         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2406
2407         return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2408                         event, status, false);
2409 }
2410
2411 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2412 {
2413         return req->request.actual == req->request.length;
2414 }
2415
2416 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2417                 const struct dwc3_event_depevt *event,
2418                 struct dwc3_request *req, int status)
2419 {
2420         int ret;
2421
2422         if (req->num_pending_sgs)
2423                 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2424                                 status);
2425         else
2426                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2427                                 status);
2428
2429         if (req->unaligned || req->zero) {
2430                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2431                                 status);
2432                 req->unaligned = false;
2433                 req->zero = false;
2434         }
2435
2436         req->request.actual = req->request.length - req->remaining;
2437
2438         if (!dwc3_gadget_ep_request_completed(req) &&
2439                         req->num_pending_sgs) {
2440                 __dwc3_gadget_kick_transfer(dep);
2441                 goto out;
2442         }
2443
2444         dwc3_gadget_giveback(dep, req, status);
2445
2446 out:
2447         return ret;
2448 }
2449
2450 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2451                 const struct dwc3_event_depevt *event, int status)
2452 {
2453         struct dwc3_request     *req;
2454         struct dwc3_request     *tmp;
2455
2456         list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2457                 int ret;
2458
2459                 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2460                                 req, status);
2461                 if (ret)
2462                         break;
2463         }
2464 }
2465
2466 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2467                 const struct dwc3_event_depevt *event)
2468 {
2469         dep->frame_number = event->parameters;
2470 }
2471
2472 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2473                 const struct dwc3_event_depevt *event)
2474 {
2475         struct dwc3             *dwc = dep->dwc;
2476         unsigned                status = 0;
2477         bool                    stop = false;
2478
2479         dwc3_gadget_endpoint_frame_from_event(dep, event);
2480
2481         if (event->status & DEPEVT_STATUS_BUSERR)
2482                 status = -ECONNRESET;
2483
2484         if ((event->status & DEPEVT_STATUS_MISSED_ISOC) &&
2485             usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2486                 status = -EXDEV;
2487
2488                 if (list_empty(&dep->started_list))
2489                         stop = true;
2490         }
2491
2492         dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2493
2494         if (dep->stream_capable && !list_empty(&dep->started_list))
2495                 __dwc3_gadget_kick_transfer(dep);
2496
2497         if (stop) {
2498                 dwc3_stop_active_transfer(dep, true);
2499                 dep->flags = DWC3_EP_ENABLED;
2500         }
2501
2502         /*
2503          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2504          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2505          */
2506         if (dwc->revision < DWC3_REVISION_183A) {
2507                 u32             reg;
2508                 int             i;
2509
2510                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2511                         dep = dwc->eps[i];
2512
2513                         if (!(dep->flags & DWC3_EP_ENABLED))
2514                                 continue;
2515
2516                         if (!list_empty(&dep->started_list))
2517                                 return;
2518                 }
2519
2520                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2521                 reg |= dwc->u1u2;
2522                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2523
2524                 dwc->u1u2 = 0;
2525         }
2526 }
2527
2528 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2529                 const struct dwc3_event_depevt *event)
2530 {
2531         dwc3_gadget_endpoint_frame_from_event(dep, event);
2532         __dwc3_gadget_start_isoc(dep);
2533 }
2534
2535 static void dwc3_endpoint_stream_event(struct dwc3 *dwc,
2536                                        const struct dwc3_event_depevt *event)
2537 {
2538         struct dwc3_ep          *dep;
2539         struct dwc3_request     *req;
2540         u8                      epnum = event->endpoint_number;
2541         u8                      stream_id;
2542
2543         dep = dwc->eps[epnum];
2544
2545         stream_id = event->parameters;
2546
2547         /* Check for request matching the streamid and delete the timer */
2548         list_for_each_entry(req, &dep->started_list, list) {
2549                 if (req->request.stream_id == stream_id) {
2550                         if (timer_pending(&req->stream_timeout_timer))
2551                                 del_timer(&req->stream_timeout_timer);
2552                         break;
2553                 }
2554         }
2555 }
2556
2557 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2558                 const struct dwc3_event_depevt *event)
2559 {
2560         struct dwc3_ep          *dep;
2561         u8                      epnum = event->endpoint_number;
2562         u8                      cmd;
2563
2564         dep = dwc->eps[epnum];
2565
2566         if (!(dep->flags & DWC3_EP_ENABLED)) {
2567                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2568                         return;
2569
2570                 /* Handle only EPCMDCMPLT when EP disabled */
2571                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2572                         return;
2573         }
2574
2575         if (epnum == 0 || epnum == 1) {
2576                 dwc3_ep0_interrupt(dwc, event);
2577                 return;
2578         }
2579
2580         switch (event->endpoint_event) {
2581         case DWC3_DEPEVT_XFERCOMPLETE:
2582                 if (!dep->stream_capable)
2583                         break;
2584                 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2585                 /* Fall Through */
2586         case DWC3_DEPEVT_XFERINPROGRESS:
2587                 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2588                 break;
2589         case DWC3_DEPEVT_XFERNOTREADY:
2590                 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2591                 break;
2592         case DWC3_DEPEVT_STREAMEVT:
2593                 if (event->status == DEPEVT_STREAMEVT_FOUND)
2594                         dwc3_endpoint_stream_event(dwc, event);
2595                 break;
2596         case DWC3_DEPEVT_EPCMDCMPLT:
2597                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2598
2599                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2600                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2601                         wake_up(&dep->wait_end_transfer);
2602                 }
2603                 break;
2604         case DWC3_DEPEVT_RXTXFIFOEVT:
2605                 break;
2606         }
2607 }
2608
2609 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2610 {
2611         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2612                 spin_unlock(&dwc->lock);
2613                 dwc->gadget_driver->disconnect(&dwc->gadget);
2614                 spin_lock(&dwc->lock);
2615         }
2616 }
2617
2618 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2619 {
2620         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2621                 spin_unlock(&dwc->lock);
2622                 dwc->gadget_driver->suspend(&dwc->gadget);
2623                 spin_lock(&dwc->lock);
2624         }
2625 }
2626
2627 static void dwc3_resume_gadget(struct dwc3 *dwc)
2628 {
2629         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2630                 spin_unlock(&dwc->lock);
2631                 dwc->gadget_driver->resume(&dwc->gadget);
2632                 spin_lock(&dwc->lock);
2633         }
2634 }
2635
2636 static void dwc3_reset_gadget(struct dwc3 *dwc)
2637 {
2638         if (!dwc->gadget_driver)
2639                 return;
2640
2641         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2642                 spin_unlock(&dwc->lock);
2643                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2644                 spin_lock(&dwc->lock);
2645         }
2646 }
2647
2648 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force)
2649 {
2650         struct dwc3 *dwc = dep->dwc;
2651         struct dwc3_gadget_ep_cmd_params params;
2652         u32 cmd;
2653         int ret;
2654
2655         if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2656             !dep->resource_index)
2657                 return;
2658
2659         /*
2660          * NOTICE: We are violating what the Databook says about the
2661          * EndTransfer command. Ideally we would _always_ wait for the
2662          * EndTransfer Command Completion IRQ, but that's causing too
2663          * much trouble synchronizing between us and gadget driver.
2664          *
2665          * We have discussed this with the IP Provider and it was
2666          * suggested to giveback all requests here, but give HW some
2667          * extra time to synchronize with the interconnect. We're using
2668          * an arbitrary 100us delay for that.
2669          *
2670          * Note also that a similar handling was tested by Synopsys
2671          * (thanks a lot Paul) and nothing bad has come out of it.
2672          * In short, what we're doing is:
2673          *
2674          * - Issue EndTransfer WITH CMDIOC bit set
2675          * - Wait 100us
2676          *
2677          * As of IP version 3.10a of the DWC_usb3 IP, the controller
2678          * supports a mode to work around the above limitation. The
2679          * software can poll the CMDACT bit in the DEPCMD register
2680          * after issuing a EndTransfer command. This mode is enabled
2681          * by writing GUCTL2[14]. This polling is already done in the
2682          * dwc3_send_gadget_ep_cmd() function so if the mode is
2683          * enabled, the EndTransfer command will have completed upon
2684          * returning from this function and we don't need to delay for
2685          * 100us.
2686          *
2687          * This mode is NOT available on the DWC_usb31 IP.
2688          */
2689
2690         cmd = DWC3_DEPCMD_ENDTRANSFER;
2691         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2692         cmd |= DWC3_DEPCMD_CMDIOC;
2693         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2694         memset(&params, 0, sizeof(params));
2695         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2696         WARN_ON_ONCE(ret);
2697         dep->resource_index = 0;
2698
2699         /*
2700          * when transfer is stopped with force rm bit false, it can be
2701          * restarted by passing resource_index in params; don't loose it
2702          */
2703         if (force)
2704                 dep->resource_index = 0;
2705
2706         if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2707                 /*
2708                  * CMD COMPLETE interrupt is not getting generated for isoc
2709                  * endpoints, so don't set DWC3_EP_END_TRANSFER_PENDING flag
2710                  */
2711                 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc))
2712                         dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2713
2714                 udelay(100);
2715         }
2716 }
2717
2718 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2719 {
2720         u32 epnum;
2721
2722         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2723                 struct dwc3_ep *dep;
2724                 int ret;
2725
2726                 dep = dwc->eps[epnum];
2727                 if (!dep)
2728                         continue;
2729
2730                 if (!(dep->flags & DWC3_EP_STALL))
2731                         continue;
2732
2733                 dep->flags &= ~DWC3_EP_STALL;
2734
2735                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2736                 WARN_ON_ONCE(ret);
2737         }
2738 }
2739
2740 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2741 {
2742         int                     reg;
2743
2744         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2745         reg &= ~DWC3_DCTL_INITU1ENA;
2746         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2747
2748         reg &= ~DWC3_DCTL_INITU2ENA;
2749         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2750
2751         dwc3_disconnect_gadget(dwc);
2752
2753         /* In USB 2.0, to avoid hibernation interrupt at the time of connection
2754          * clear DWC3_DCTL_KEEP_CONNECT bit.
2755          */
2756         if (dwc->has_hibernation) {
2757                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2758                 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2759                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2760         }
2761
2762         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2763         dwc->setup_packet_pending = false;
2764         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2765
2766         dwc->connected = false;
2767 }
2768
2769 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2770 {
2771         u32                     reg;
2772
2773         dwc->connected = true;
2774
2775         /*
2776          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2777          * would cause a missing Disconnect Event if there's a
2778          * pending Setup Packet in the FIFO.
2779          *
2780          * There's no suggested workaround on the official Bug
2781          * report, which states that "unless the driver/application
2782          * is doing any special handling of a disconnect event,
2783          * there is no functional issue".
2784          *
2785          * Unfortunately, it turns out that we _do_ some special
2786          * handling of a disconnect event, namely complete all
2787          * pending transfers, notify gadget driver of the
2788          * disconnection, and so on.
2789          *
2790          * Our suggested workaround is to follow the Disconnect
2791          * Event steps here, instead, based on a setup_packet_pending
2792          * flag. Such flag gets set whenever we have a SETUP_PENDING
2793          * status for EP0 TRBs and gets cleared on XferComplete for the
2794          * same endpoint.
2795          *
2796          * Refers to:
2797          *
2798          * STAR#9000466709: RTL: Device : Disconnect event not
2799          * generated if setup packet pending in FIFO
2800          */
2801         if (dwc->revision < DWC3_REVISION_188A) {
2802                 if (dwc->setup_packet_pending)
2803                         dwc3_gadget_disconnect_interrupt(dwc);
2804         }
2805
2806         dwc3_reset_gadget(dwc);
2807
2808         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2809         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2810         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2811         dwc->test_mode = false;
2812         dwc3_clear_stall_all_ep(dwc);
2813
2814         /* Reset device address to zero */
2815         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2816         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2817         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2818 }
2819
2820 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2821 {
2822         struct dwc3_ep          *dep;
2823         int                     ret;
2824         u32                     reg;
2825         u8                      speed;
2826
2827         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2828         speed = reg & DWC3_DSTS_CONNECTSPD;
2829         dwc->speed = speed;
2830
2831         /*
2832          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2833          * each time on Connect Done.
2834          *
2835          * Currently we always use the reset value. If any platform
2836          * wants to set this to a different value, we need to add a
2837          * setting and update GCTL.RAMCLKSEL here.
2838          */
2839
2840         switch (speed) {
2841         case DWC3_DSTS_SUPERSPEED_PLUS:
2842                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2843                 dwc->gadget.ep0->maxpacket = 512;
2844                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2845                 break;
2846         case DWC3_DSTS_SUPERSPEED:
2847                 /*
2848                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2849                  * would cause a missing USB3 Reset event.
2850                  *
2851                  * In such situations, we should force a USB3 Reset
2852                  * event by calling our dwc3_gadget_reset_interrupt()
2853                  * routine.
2854                  *
2855                  * Refers to:
2856                  *
2857                  * STAR#9000483510: RTL: SS : USB3 reset event may
2858                  * not be generated always when the link enters poll
2859                  */
2860                 if (dwc->revision < DWC3_REVISION_190A)
2861                         dwc3_gadget_reset_interrupt(dwc);
2862
2863                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2864                 dwc->gadget.ep0->maxpacket = 512;
2865                 dwc->gadget.speed = USB_SPEED_SUPER;
2866                 break;
2867         case DWC3_DSTS_HIGHSPEED:
2868                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2869                 dwc->gadget.ep0->maxpacket = 64;
2870                 dwc->gadget.speed = USB_SPEED_HIGH;
2871                 break;
2872         case DWC3_DSTS_FULLSPEED:
2873                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2874                 dwc->gadget.ep0->maxpacket = 64;
2875                 dwc->gadget.speed = USB_SPEED_FULL;
2876                 break;
2877         case DWC3_DSTS_LOWSPEED:
2878                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2879                 dwc->gadget.ep0->maxpacket = 8;
2880                 dwc->gadget.speed = USB_SPEED_LOW;
2881                 break;
2882         }
2883
2884         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2885
2886         /* Enable USB2 LPM Capability */
2887
2888         if ((dwc->revision > DWC3_REVISION_194A) &&
2889             (speed != DWC3_DSTS_SUPERSPEED) &&
2890             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2891                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2892                 reg |= DWC3_DCFG_LPM_CAP;
2893                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2894
2895                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2896                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2897
2898                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2899
2900                 /*
2901                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2902                  * DCFG.LPMCap is set, core responses with an ACK and the
2903                  * BESL value in the LPM token is less than or equal to LPM
2904                  * NYET threshold.
2905                  */
2906                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2907                                 && dwc->has_lpm_erratum,
2908                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2909
2910                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2911                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2912
2913                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2914         } else {
2915                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2916                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2917                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2918         }
2919
2920         dep = dwc->eps[0];
2921         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2922         if (ret) {
2923                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2924                 return;
2925         }
2926
2927         dep = dwc->eps[1];
2928         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2929         if (ret) {
2930                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2931                 return;
2932         }
2933
2934         /*
2935          * In USB 2.0, to avoid hibernation interrupt at the time of connection
2936          * set DWC3_DCTL_KEEP_CONNECT bit here
2937          */
2938         if (dwc->has_hibernation) {
2939                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2940                 reg |= DWC3_DCTL_KEEP_CONNECT;
2941                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2942         }
2943
2944         /*
2945          * Configure PHY via GUSB3PIPECTLn if required.
2946          *
2947          * Update GTXFIFOSIZn
2948          *
2949          * In both cases reset values should be sufficient.
2950          */
2951 }
2952
2953 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2954 {
2955         /*
2956          * TODO take core out of low power mode when that's
2957          * implemented.
2958          */
2959
2960         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2961                 spin_unlock(&dwc->lock);
2962                 dwc->gadget_driver->resume(&dwc->gadget);
2963                 spin_lock(&dwc->lock);
2964         }
2965 }
2966
2967 static irqreturn_t wakeup_interrupt(int irq, void *_dwc)
2968 {
2969         struct dwc3 *dwc = (struct dwc3 *)_dwc;
2970
2971         spin_lock(&dwc->lock);
2972         gadget_wakeup_interrupt(dwc);
2973         spin_unlock(&dwc->lock);
2974
2975         return IRQ_HANDLED;
2976 }
2977
2978 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2979                 unsigned int evtinfo)
2980 {
2981         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2982         unsigned int            pwropt;
2983
2984         /*
2985          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2986          * Hibernation mode enabled which would show up when device detects
2987          * host-initiated U3 exit.
2988          *
2989          * In that case, device will generate a Link State Change Interrupt
2990          * from U3 to RESUME which is only necessary if Hibernation is
2991          * configured in.
2992          *
2993          * There are no functional changes due to such spurious event and we
2994          * just need to ignore it.
2995          *
2996          * Refers to:
2997          *
2998          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2999          * operational mode
3000          */
3001         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3002         if ((dwc->revision < DWC3_REVISION_250A) &&
3003                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3004                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3005                                 (next == DWC3_LINK_STATE_RESUME)) {
3006                         return;
3007                 }
3008         }
3009
3010         /*
3011          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
3012          * on the link partner, the USB session might do multiple entry/exit
3013          * of low power states before a transfer takes place.
3014          *
3015          * Due to this problem, we might experience lower throughput. The
3016          * suggested workaround is to disable DCTL[12:9] bits if we're
3017          * transitioning from U1/U2 to U0 and enable those bits again
3018          * after a transfer completes and there are no pending transfers
3019          * on any of the enabled endpoints.
3020          *
3021          * This is the first half of that workaround.
3022          *
3023          * Refers to:
3024          *
3025          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
3026          * core send LGO_Ux entering U0
3027          */
3028         if (dwc->revision < DWC3_REVISION_183A) {
3029                 if (next == DWC3_LINK_STATE_U0) {
3030                         u32     u1u2;
3031                         u32     reg;
3032
3033                         switch (dwc->link_state) {
3034                         case DWC3_LINK_STATE_U1:
3035                         case DWC3_LINK_STATE_U2:
3036                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3037                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3038                                                 | DWC3_DCTL_ACCEPTU2ENA
3039                                                 | DWC3_DCTL_INITU1ENA
3040                                                 | DWC3_DCTL_ACCEPTU1ENA);
3041
3042                                 if (!dwc->u1u2)
3043                                         dwc->u1u2 = reg & u1u2;
3044
3045                                 reg &= ~u1u2;
3046
3047                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3048                                 break;
3049                         default:
3050                                 /* do nothing */
3051                                 break;
3052                         }
3053                 }
3054         }
3055
3056         switch (next) {
3057         case DWC3_LINK_STATE_U1:
3058                 if (dwc->speed == USB_SPEED_SUPER)
3059                         dwc3_suspend_gadget(dwc);
3060                 break;
3061         case DWC3_LINK_STATE_U2:
3062         case DWC3_LINK_STATE_U3:
3063                 dwc3_suspend_gadget(dwc);
3064                 break;
3065         case DWC3_LINK_STATE_RESUME:
3066                 dwc3_resume_gadget(dwc);
3067                 break;
3068         default:
3069                 /* do nothing */
3070                 break;
3071         }
3072
3073         dwc->link_state = next;
3074 }
3075
3076 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3077                                           unsigned int evtinfo)
3078 {
3079         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3080
3081         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3082                 dwc3_suspend_gadget(dwc);
3083
3084         dwc->link_state = next;
3085 }
3086
3087 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3088                 unsigned int evtinfo)
3089 {
3090         unsigned int is_ss = evtinfo & BIT(4);
3091
3092         /*
3093          * WORKAROUND: DWC3 revison 2.20a with hibernation support
3094          * have a known issue which can cause USB CV TD.9.23 to fail
3095          * randomly.
3096          *
3097          * Because of this issue, core could generate bogus hibernation
3098          * events which SW needs to ignore.
3099          *
3100          * Refers to:
3101          *
3102          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
3103          * Device Fallback from SuperSpeed
3104          */
3105         if ((!!is_ss ^ (dwc->speed >= DWC3_DSTS_SUPERSPEED)) &&
3106             (!(dwc->has_hibernation)))
3107                 return;
3108
3109         /* enter hibernation here */
3110         gadget_hibernation_interrupt(dwc);
3111 }
3112
3113 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3114                 const struct dwc3_event_devt *event)
3115 {
3116         switch (event->type) {
3117         case DWC3_DEVICE_EVENT_DISCONNECT:
3118                 dwc3_gadget_disconnect_interrupt(dwc);
3119                 break;
3120         case DWC3_DEVICE_EVENT_RESET:
3121                 dwc3_gadget_reset_interrupt(dwc);
3122                 break;
3123         case DWC3_DEVICE_EVENT_CONNECT_DONE:
3124                 dwc3_gadget_conndone_interrupt(dwc);
3125                 break;
3126         case DWC3_DEVICE_EVENT_WAKEUP:
3127                 dwc3_gadget_wakeup_interrupt(dwc);
3128                 break;
3129         case DWC3_DEVICE_EVENT_HIBER_REQ:
3130                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3131                                         "unexpected hibernation event\n"))
3132                         break;
3133
3134                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3135                 break;
3136         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3137                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3138                 break;
3139         case DWC3_DEVICE_EVENT_EOPF:
3140                 /* It changed to be suspend event for version 2.30a and above */
3141                 if (dwc->revision >= DWC3_REVISION_230A) {
3142                         /*
3143                          * Ignore suspend event until the gadget enters into
3144                          * USB_STATE_CONFIGURED state.
3145                          */
3146                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3147                                 dwc3_gadget_suspend_interrupt(dwc,
3148                                                 event->event_info);
3149                 }
3150                 break;
3151         case DWC3_DEVICE_EVENT_SOF:
3152         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3153         case DWC3_DEVICE_EVENT_CMD_CMPL:
3154         case DWC3_DEVICE_EVENT_OVERFLOW:
3155                 break;
3156         default:
3157                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3158         }
3159 }
3160
3161 static void dwc3_process_event_entry(struct dwc3 *dwc,
3162                 const union dwc3_event *event)
3163 {
3164         trace_dwc3_event(event->raw, dwc);
3165
3166         if (!event->type.is_devspec)
3167                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3168         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3169                 dwc3_gadget_interrupt(dwc, &event->devt);
3170         else
3171                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3172 }
3173
3174 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3175 {
3176         struct dwc3 *dwc = evt->dwc;
3177         irqreturn_t ret = IRQ_NONE;
3178         int left;
3179         u32 reg;
3180
3181         left = evt->count;
3182
3183         if (!(evt->flags & DWC3_EVENT_PENDING))
3184                 return IRQ_NONE;
3185
3186         while (left > 0) {
3187                 union dwc3_event event;
3188
3189                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3190
3191                 dwc3_process_event_entry(dwc, &event);
3192
3193                 /*
3194                  * FIXME we wrap around correctly to the next entry as
3195                  * almost all entries are 4 bytes in size. There is one
3196                  * entry which has 12 bytes which is a regular entry
3197                  * followed by 8 bytes data. ATM I don't know how
3198                  * things are organized if we get next to the a
3199                  * boundary so I worry about that once we try to handle
3200                  * that.
3201                  */
3202                 evt->lpos = (evt->lpos + 4) % evt->length;
3203                 left -= 4;
3204
3205                 if (dwc->is_hibernated)
3206                         break;
3207         }
3208
3209         evt->count = 0;
3210         evt->flags &= ~DWC3_EVENT_PENDING;
3211         ret = IRQ_HANDLED;
3212
3213         if (dwc->is_hibernated)
3214                 return ret;
3215
3216         /* Unmask interrupt */
3217         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3218         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3219         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3220
3221         if (dwc->imod_interval) {
3222                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3223                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3224         }
3225
3226         return ret;
3227 }
3228
3229 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3230 {
3231         struct dwc3_event_buffer *evt = _evt;
3232         struct dwc3 *dwc = evt->dwc;
3233         unsigned long flags;
3234         irqreturn_t ret = IRQ_NONE;
3235
3236         spin_lock_irqsave(&dwc->lock, flags);
3237         ret = dwc3_process_event_buf(evt);
3238         spin_unlock_irqrestore(&dwc->lock, flags);
3239
3240         return ret;
3241 }
3242
3243 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3244 {
3245         struct dwc3 *dwc = evt->dwc;
3246         u32 amount;
3247         u32 count;
3248         u32 reg;
3249
3250         if (pm_runtime_suspended(dwc->dev)) {
3251                 pm_runtime_get(dwc->dev);
3252                 disable_irq_nosync(dwc->irq_gadget);
3253                 dwc->pending_events = true;
3254                 return IRQ_HANDLED;
3255         }
3256
3257         if (dwc->is_hibernated)
3258                 return IRQ_HANDLED;
3259
3260         /*
3261          * With PCIe legacy interrupt, test shows that top-half irq handler can
3262          * be called again after HW interrupt deassertion. Check if bottom-half
3263          * irq event handler completes before caching new event to prevent
3264          * losing events.
3265          */
3266         if (evt->flags & DWC3_EVENT_PENDING)
3267                 return IRQ_HANDLED;
3268
3269         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3270         count &= DWC3_GEVNTCOUNT_MASK;
3271         if (!count)
3272                 return IRQ_NONE;
3273
3274         evt->count = count;
3275         evt->flags |= DWC3_EVENT_PENDING;
3276
3277         /* Mask interrupt */
3278         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3279         reg |= DWC3_GEVNTSIZ_INTMASK;
3280         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3281
3282         amount = min(count, evt->length - evt->lpos);
3283         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3284
3285         if (amount < count)
3286                 memcpy(evt->cache, evt->buf, count - amount);
3287
3288         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3289
3290         return IRQ_WAKE_THREAD;
3291 }
3292
3293 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3294 {
3295         struct dwc3_event_buffer        *evt = _evt;
3296
3297         return dwc3_check_event_buf(evt);
3298 }
3299
3300 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3301 {
3302         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3303         int irq;
3304
3305         irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3306         if (irq > 0)
3307                 goto out;
3308
3309         if (irq == -EPROBE_DEFER)
3310                 goto out;
3311
3312         irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3313         if (irq > 0)
3314                 goto out;
3315
3316         if (irq == -EPROBE_DEFER)
3317                 goto out;
3318
3319         irq = platform_get_irq(dwc3_pdev, 0);
3320         if (irq > 0)
3321                 dwc->irq_gadget = irq;
3322
3323         if (irq == -EPROBE_DEFER)
3324                 goto out;
3325
3326         /* look for wakeup interrupt if hibernation is supported */
3327         if (dwc->has_hibernation) {
3328                 irq = platform_get_irq(dwc3_pdev, 2);
3329                 if (irq > 0)
3330                         dwc->irq_wakeup = irq;
3331
3332                 if (irq == -EPROBE_DEFER)
3333                         goto out;
3334         }
3335
3336         if (irq <= 0) {
3337                 if (irq != -EPROBE_DEFER)
3338                         dev_err(dwc->dev, "missing peripheral IRQ\n");
3339
3340                 if (!irq)
3341                         irq = -EINVAL;
3342         }
3343 out:
3344         return irq;
3345 }
3346
3347 /**
3348  * dwc3_gadget_init - initializes gadget related registers
3349  * @dwc: pointer to our controller context structure
3350  *
3351  * Returns 0 on success otherwise negative errno.
3352  */
3353 int dwc3_gadget_init(struct dwc3 *dwc)
3354 {
3355         int ret;
3356         int irq;
3357
3358         irq = dwc3_gadget_get_irq(dwc);
3359         if (irq < 0) {
3360                 ret = irq;
3361                 goto err0;
3362         }
3363
3364         if (dwc->dr_mode == USB_DR_MODE_OTG) {
3365                 struct usb_phy *phy;
3366
3367                 /* Switch otg to peripheral mode */
3368                 phy = usb_get_phy(USB_PHY_TYPE_USB3);
3369                 if (!IS_ERR(phy)) {
3370                         if (phy && phy->otg)
3371                                 otg_set_peripheral(phy->otg,
3372                                                 (struct usb_gadget *)(long)1);
3373                         usb_put_phy(phy);
3374                 }
3375         }
3376
3377         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3378                                           sizeof(*dwc->ep0_trb) * 2,
3379                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3380         if (!dwc->ep0_trb) {
3381                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3382                 ret = -ENOMEM;
3383                 goto err0;
3384         }
3385
3386         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3387         if (!dwc->setup_buf) {
3388                 ret = -ENOMEM;
3389                 goto err1;
3390         }
3391
3392         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3393                         &dwc->bounce_addr, GFP_KERNEL);
3394         if (!dwc->bounce) {
3395                 ret = -ENOMEM;
3396                 goto err2;
3397         }
3398
3399         init_completion(&dwc->ep0_in_setup);
3400
3401         dwc->gadget.ops                 = &dwc3_gadget_ops;
3402         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3403         dwc->gadget.sg_supported        = true;
3404         dwc->gadget.name                = "dwc3-gadget";
3405         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
3406
3407         /*
3408          * FIXME We might be setting max_speed to <SUPER, however versions
3409          * <2.20a of dwc3 have an issue with metastability (documented
3410          * elsewhere in this driver) which tells us we can't set max speed to
3411          * anything lower than SUPER.
3412          *
3413          * Because gadget.max_speed is only used by composite.c and function
3414          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3415          * to happen so we avoid sending SuperSpeed Capability descriptor
3416          * together with our BOS descriptor as that could confuse host into
3417          * thinking we can handle super speed.
3418          *
3419          * Note that, in fact, we won't even support GetBOS requests when speed
3420          * is less than super speed because we don't have means, yet, to tell
3421          * composite.c that we are USB 2.0 + LPM ECN.
3422          */
3423         if (dwc->revision < DWC3_REVISION_220A &&
3424             !dwc->dis_metastability_quirk)
3425                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3426                                 dwc->revision);
3427
3428         dwc->gadget.max_speed           = dwc->maximum_speed;
3429
3430         /*
3431          * REVISIT: Here we should clear all pending IRQs to be
3432          * sure we're starting from a well known location.
3433          */
3434
3435         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3436         if (ret)
3437                 goto err3;
3438
3439         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3440         if (ret) {
3441                 dev_err(dwc->dev, "failed to register udc\n");
3442                 goto err4;
3443         }
3444
3445         if (dwc->dr_mode == USB_DR_MODE_OTG) {
3446                 struct usb_phy *phy;
3447
3448                 phy = usb_get_phy(USB_PHY_TYPE_USB3);
3449                 if (!IS_ERR(phy)) {
3450                         if (phy && phy->otg) {
3451                                 ret = otg_set_peripheral(phy->otg,
3452                                                 &dwc->gadget);
3453                                 if (ret) {
3454                                         dev_err(dwc->dev,
3455                                         "%s otg_set_peripheral failed\n",
3456                                                         __func__);
3457                                         usb_put_phy(phy);
3458                                         phy = NULL;
3459                                         goto err4;
3460                                 }
3461                         } else {
3462                                 usb_put_phy(phy);
3463                                 phy = NULL;
3464                         }
3465                 }
3466         }
3467
3468         return 0;
3469
3470 err4:
3471         dwc3_gadget_free_endpoints(dwc);
3472
3473 err3:
3474         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3475                         dwc->bounce_addr);
3476
3477 err2:
3478         kfree(dwc->setup_buf);
3479
3480 err1:
3481         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3482                         dwc->ep0_trb, dwc->ep0_trb_addr);
3483
3484 err0:
3485         return ret;
3486 }
3487
3488 /* -------------------------------------------------------------------------- */
3489
3490 void dwc3_gadget_exit(struct dwc3 *dwc)
3491 {
3492         usb_del_gadget_udc(&dwc->gadget);
3493         dwc3_gadget_free_endpoints(dwc);
3494         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3495                           dwc->bounce_addr);
3496         kfree(dwc->setup_buf);
3497         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3498                           dwc->ep0_trb, dwc->ep0_trb_addr);
3499 }
3500
3501 int dwc3_gadget_suspend(struct dwc3 *dwc)
3502 {
3503         if (!dwc->gadget_driver)
3504                 return 0;
3505
3506         if (dwc->is_hibernated) {
3507                 /*
3508                  * As we are about to suspend, wake the controller from
3509                  * D3 & hibernation states
3510                  */
3511                 dwc->force_hiber_wake = true;
3512                 gadget_wakeup_interrupt(dwc);
3513                 dwc->force_hiber_wake = false;
3514         }
3515
3516         dwc3_gadget_run_stop(dwc, false, false);
3517         dwc3_disconnect_gadget(dwc);
3518         __dwc3_gadget_stop(dwc);
3519
3520         return 0;
3521 }
3522
3523 int dwc3_gadget_resume(struct dwc3 *dwc)
3524 {
3525         int                     ret;
3526         u32                     reg;
3527
3528         if (!dwc->gadget_driver)
3529                 return 0;
3530
3531         ret = __dwc3_gadget_start(dwc);
3532         if (ret < 0)
3533                 goto err0;
3534
3535         ret = dwc3_gadget_run_stop(dwc, true, false);
3536         if (ret < 0)
3537                 goto err1;
3538
3539         /* In USB 2.0, to avoid hibernation interrupt at the time of connection
3540          * set DWC3_DCTL_KEEP_CONNECT bit.
3541          */
3542         if (dwc->has_hibernation) {
3543                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3544                 reg |= DWC3_DCTL_KEEP_CONNECT;
3545                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3546         }
3547
3548         return 0;
3549
3550 err1:
3551         __dwc3_gadget_stop(dwc);
3552
3553 err0:
3554         return ret;
3555 }
3556
3557 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3558 {
3559         if (dwc->pending_events) {
3560                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3561                 dwc->pending_events = false;
3562                 enable_irq(dwc->irq_gadget);
3563         }
3564 }