drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 #define DWC3_ALIGN_FRAME(d)     (((d)->frame_number + (d)->interval) \
  31                                         & ~((d)->interval - 1))
  32
  33 /**
  34  * dwc3_gadget_set_test_mode - enables usb2 test modes
  35  * @dwc: pointer to our context structure
  36  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  37  *
  38  * Caller should take care of locking. This function will return 0 on
  39  * success or -EINVAL if wrong Test Selector is passed.
  40  */
  41 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  42 {
  43         u32             reg;
  44
  45         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  46         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  47
  48         switch (mode) {
  49         case TEST_J:
  50         case TEST_K:
  51         case TEST_SE0_NAK:
  52         case TEST_PACKET:
  53         case TEST_FORCE_EN:
  54                 reg |= mode << 1;
  55                 break;
  56         default:
  57                 return -EINVAL;
  58         }
  59
  60         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  61
  62         return 0;
  63 }
  64
  65 /**
  66  * dwc3_gadget_get_link_state - gets current state of usb link
  67  * @dwc: pointer to our context structure
  68  *
  69  * Caller should take care of locking. This function will
  70  * return the link state on success (>= 0) or -ETIMEDOUT.
  71  */
  72 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  73 {
  74         u32             reg;
  75
  76         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  77
  78         return DWC3_DSTS_USBLNKST(reg);
  79 }
  80
  81 /**
  82  * dwc3_gadget_set_link_state - sets usb link to a particular state
  83  * @dwc: pointer to our context structure
  84  * @state: the state to put link into
  85  *
  86  * Caller should take care of locking. This function will
  87  * return 0 on success or -ETIMEDOUT.
  88  */
  89 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  90 {
  91         int             retries = 10000;
  92         u32             reg;
  93
  94         /*
  95          * Wait until device controller is ready. Only applies to 1.94a and
  96          * later RTL.
  97          */
  98         if (dwc->revision >= DWC3_REVISION_194A) {
  99                 while (--retries) {
 100                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 101                         if (reg & DWC3_DSTS_DCNRD)
 102                                 udelay(5);
 103                         else
 104                                 break;
 105                 }
 106
 107                 if (retries <= 0)
 108                         return -ETIMEDOUT;
 109         }
 110
 111         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 112         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 113
 114         /* set requested state */
 115         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 116         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 117
 118         /*
 119          * The following code is racy when called from dwc3_gadget_wakeup,
 120          * and is not needed, at least on newer versions
 121          */
 122         if (dwc->revision >= DWC3_REVISION_194A)
 123                 return 0;
 124
 125         /* wait for a change in DSTS */
 126         retries = 10000;
 127         while (--retries) {
 128                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 129
 130                 if (DWC3_DSTS_USBLNKST(reg) == state)
 131                         return 0;
 132
 133                 udelay(5);
 134         }
 135
 136         return -ETIMEDOUT;
 137 }
 138
 139 /**
 140  * dwc3_ep_inc_trb - increment a trb index.
 141  * @index: Pointer to the TRB index to increment.
 142  *
 143  * The index should never point to the link TRB. After incrementing,
 144  * if it is point to the link TRB, wrap around to the beginning. The
 145  * link TRB is always at the last TRB entry.
 146  */
 147 static void dwc3_ep_inc_trb(u8 *index)
 148 {
 149         (*index)++;
 150         if (*index == (DWC3_TRB_NUM - 1))
 151                 *index = 0;
 152 }
 153
 154 /**
 155  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 156  * @dep: The endpoint whose enqueue pointer we're incrementing
 157  */
 158 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 159 {
 160         dwc3_ep_inc_trb(&dep->trb_enqueue);
 161 }
 162
 163 /**
 164  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 165  * @dep: The endpoint whose enqueue pointer we're incrementing
 166  */
 167 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 168 {
 169         dwc3_ep_inc_trb(&dep->trb_dequeue);
 170 }
 171
 172 static void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 173                 struct dwc3_request *req, int status)
 174 {
 175         struct dwc3                     *dwc = dep->dwc;
 176
 177         req->started = false;
 178         list_del(&req->list);
 179         req->remaining = 0;
 180
 181         if (req->request.status == -EINPROGRESS)
 182                 req->request.status = status;
 183
 184         if (req->trb)
 185                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 186                                 &req->request, req->direction);
 187
 188         req->trb = NULL;
 189         trace_dwc3_gadget_giveback(req);
 190
 191         if (dep->number > 1)
 192                 pm_runtime_put(dwc->dev);
 193 }
 194
 195 /**
 196  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 197  * @dep: The endpoint to whom the request belongs to
 198  * @req: The request we're giving back
 199  * @status: completion code for the request
 200  *
 201  * Must be called with controller's lock held and interrupts disabled. This
 202  * function will unmap @req and call its ->complete() callback to notify upper
 203  * layers that it has completed.
 204  */
 205 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 206                 int status)
 207 {
 208         struct dwc3                     *dwc = dep->dwc;
 209
 210         if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
 211                 del_timer(&req->stream_timeout_timer);
 212
 213         dwc3_gadget_del_and_unmap_request(dep, req, status);
 214
 215         spin_unlock(&dwc->lock);
 216         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 217         spin_lock(&dwc->lock);
 218 }
 219
 220 /**
 221  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 222  * @dwc: pointer to the controller context
 223  * @cmd: the command to be issued
 224  * @param: command parameter
 225  *
 226  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 227  * and wait for its completion.
 228  */
 229 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 230 {
 231         u32             timeout = 500;
 232         int             status = 0;
 233         int             ret = 0;
 234         u32             reg;
 235
 236         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 237         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 238
 239         do {
 240                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 241                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 242                         status = DWC3_DGCMD_STATUS(reg);
 243                         if (status)
 244                                 ret = -EINVAL;
 245                         break;
 246                 }
 247         } while (--timeout);
 248
 249         if (!timeout) {
 250                 ret = -ETIMEDOUT;
 251                 status = -ETIMEDOUT;
 252         }
 253
 254         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 255
 256         return ret;
 257 }
 258
 259 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 260
 261 /**
 262  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 263  * @dep: the endpoint to which the command is going to be issued
 264  * @cmd: the command to be issued
 265  * @params: parameters to the command
 266  *
 267  * Caller should handle locking. This function will issue @cmd with given
 268  * @params to @dep and wait for its completion.
 269  */
 270 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 271                 struct dwc3_gadget_ep_cmd_params *params)
 272 {
 273         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 274         struct dwc3             *dwc = dep->dwc;
 275         u32                     timeout = 1000;
 276         u32                     reg;
 277
 278         int                     cmd_status = 0;
 279         int                     susphy = false;
 280         int                     ret = -EINVAL;
 281
 282         /*
 283          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 284          * we're issuing an endpoint command, we must check if
 285          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 286          *
 287          * We will also set SUSPHY bit to what it was before returning as stated
 288          * by the same section on Synopsys databook.
 289          */
 290         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 291                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 292                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 293                         susphy = true;
 294                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 295                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 296                 }
 297         }
 298
 299         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 300                 int             needs_wakeup;
 301
 302                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 303                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 304                                 dwc->link_state == DWC3_LINK_STATE_U3);
 305
 306                 if (unlikely(needs_wakeup)) {
 307                         ret = __dwc3_gadget_wakeup(dwc);
 308                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 309                                         ret);
 310                 }
 311         }
 312
 313         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 314         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 315         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 316
 317         /*
 318          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 319          * not relying on XferNotReady, we can make use of a special "No
 320          * Response Update Transfer" command where we should clear both CmdAct
 321          * and CmdIOC bits.
 322          *
 323          * With this, we don't need to wait for command completion and can
 324          * straight away issue further commands to the endpoint.
 325          *
 326          * NOTICE: We're making an assumption that control endpoints will never
 327          * make use of Update Transfer command. This is a safe assumption
 328          * because we can never have more than one request at a time with
 329          * Control Endpoints. If anybody changes that assumption, this chunk
 330          * needs to be updated accordingly.
 331          */
 332         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 333                         !usb_endpoint_xfer_isoc(desc))
 334                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 335         else
 336                 cmd |= DWC3_DEPCMD_CMDACT;
 337
 338         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 339         do {
 340                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 341                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 342                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 343
 344                         switch (cmd_status) {
 345                         case 0:
 346                                 ret = 0;
 347                                 break;
 348                         case DEPEVT_TRANSFER_NO_RESOURCE:
 349                                 ret = -EINVAL;
 350                                 break;
 351                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 352                                 /*
 353                                  * SW issues START TRANSFER command to
 354                                  * isochronous ep with future frame interval. If
 355                                  * future interval time has already passed when
 356                                  * core receives the command, it will respond
 357                                  * with an error status of 'Bus Expiry'.
 358                                  *
 359                                  * Instead of always returning -EINVAL, let's
 360                                  * give a hint to the gadget driver that this is
 361                                  * the case by returning -EAGAIN.
 362                                  */
 363                                 ret = -EAGAIN;
 364                                 break;
 365                         default:
 366                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 367                         }
 368
 369                         break;
 370                 }
 371         } while (--timeout);
 372
 373         if (timeout == 0) {
 374                 ret = -ETIMEDOUT;
 375                 cmd_status = -ETIMEDOUT;
 376         }
 377
 378         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 379
 380         if (ret == 0) {
 381                 switch (DWC3_DEPCMD_CMD(cmd)) {
 382                 case DWC3_DEPCMD_STARTTRANSFER:
 383                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 384                         dwc3_gadget_ep_get_transfer_index(dep);
 385                         break;
 386                 case DWC3_DEPCMD_ENDTRANSFER:
 387                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 388                         break;
 389                 default:
 390                         /* nothing */
 391                         break;
 392                 }
 393         }
 394
 395         if (unlikely(susphy)) {
 396                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 397                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 398                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 399         }
 400
 401         return ret;
 402 }
 403
 404 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 405 {
 406         struct dwc3 *dwc = dep->dwc;
 407         struct dwc3_gadget_ep_cmd_params params;
 408         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 409
 410         /*
 411          * As of core revision 2.60a the recommended programming model
 412          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 413          * command for IN endpoints. This is to prevent an issue where
 414          * some (non-compliant) hosts may not send ACK TPs for pending
 415          * IN transfers due to a mishandled error condition. Synopsys
 416          * STAR 9000614252.
 417          */
 418         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
 419             (dwc->gadget.speed >= USB_SPEED_SUPER))
 420                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 421
 422         memset(&params, 0, sizeof(params));
 423
 424         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 425 }
 426
 427 dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep, struct dwc3_trb *trb)
 428 {
 429         u32             offset = (char *) trb - (char *) dep->trb_pool;
 430
 431         return dep->trb_pool_dma + offset;
 432 }
 433
 434 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 435 {
 436         struct dwc3             *dwc = dep->dwc;
 437
 438         if (dep->trb_pool)
 439                 return 0;
 440
 441         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 442                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 443                         &dep->trb_pool_dma, GFP_KERNEL);
 444         if (!dep->trb_pool) {
 445                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 446                                 dep->name);
 447                 return -ENOMEM;
 448         }
 449
 450         return 0;
 451 }
 452
 453 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 454 {
 455         struct dwc3             *dwc = dep->dwc;
 456
 457         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 458                         dep->trb_pool, dep->trb_pool_dma);
 459
 460         dep->trb_pool = NULL;
 461         dep->trb_pool_dma = 0;
 462 }
 463
 464 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
 465 {
 466         struct dwc3_gadget_ep_cmd_params params;
 467
 468         memset(&params, 0x00, sizeof(params));
 469
 470         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 471
 472         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 473                         &params);
 474 }
 475
 476 /**
 477  * dwc3_gadget_start_config - configure ep resources
 478  * @dep: endpoint that is being enabled
 479  *
 480  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 481  * completion, it will set Transfer Resource for all available endpoints.
 482  *
 483  * The assignment of transfer resources cannot perfectly follow the data book
 484  * due to the fact that the controller driver does not have all knowledge of the
 485  * configuration in advance. It is given this information piecemeal by the
 486  * composite gadget framework after every SET_CONFIGURATION and
 487  * SET_INTERFACE. Trying to follow the databook programming model in this
 488  * scenario can cause errors. For two reasons:
 489  *
 490  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 491  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 492  * incorrect in the scenario of multiple interfaces.
 493  *
 494  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 495  * endpoint on alt setting (8.1.6).
 496  *
 497  * The following simplified method is used instead:
 498  *
 499  * All hardware endpoints can be assigned a transfer resource and this setting
 500  * will stay persistent until either a core reset or hibernation. So whenever we
 501  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 502  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 503  * guaranteed that there are as many transfer resources as endpoints.
 504  *
 505  * This function is called for each endpoint when it is being enabled but is
 506  * triggered only when called for EP0-out, which always happens first, and which
 507  * should only happen in one of the above conditions.
 508  */
 509 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 510 {
 511         struct dwc3_gadget_ep_cmd_params params;
 512         struct dwc3             *dwc;
 513         u32                     cmd;
 514         int                     i;
 515         int                     ret;
 516
 517         if (dep->number)
 518                 return 0;
 519
 520         memset(&params, 0x00, sizeof(params));
 521         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 522         dwc = dep->dwc;
 523
 524         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 525         if (ret)
 526                 return ret;
 527
 528         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 529                 struct dwc3_ep *dep = dwc->eps[i];
 530
 531                 if (!dep)
 532                         continue;
 533
 534                 ret = dwc3_gadget_set_xfer_resource(dep);
 535                 if (ret)
 536                         return ret;
 537         }
 538
 539         return 0;
 540 }
 541
 542 static void stream_timeout_function(struct timer_list *arg)
 543 {
 544         struct dwc3_request     *req = from_timer(req, arg, stream_timeout_timer);
 545         struct dwc3_ep          *dep = req->dep;
 546         struct dwc3             *dwc = dep->dwc;
 547         unsigned long           flags;
 548
 549         spin_lock_irqsave(&dwc->lock, flags);
 550         dwc3_stop_active_transfer(dep, true);
 551         __dwc3_gadget_kick_transfer(dep);
 552         spin_unlock_irqrestore(&dwc->lock, flags);
 553 }
 554
 555 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 556 {
 557         const struct usb_ss_ep_comp_descriptor *comp_desc;
 558         const struct usb_endpoint_descriptor *desc;
 559         struct dwc3_gadget_ep_cmd_params params;
 560         struct dwc3 *dwc = dep->dwc;
 561
 562         comp_desc = dep->endpoint.comp_desc;
 563         desc = dep->endpoint.desc;
 564
 565         memset(&params, 0x00, sizeof(params));
 566
 567         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 568                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 569
 570         /* Burst size is only needed in SuperSpeed mode */
 571         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 572                 u32 burst = dep->endpoint.maxburst;
 573                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 574         }
 575
 576         params.param0 |= action;
 577         if (action == DWC3_DEPCFG_ACTION_RESTORE)
 578                 params.param2 |= dep->saved_state;
 579
 580         if (usb_endpoint_xfer_control(desc))
 581                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 582
 583         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 584                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 585
 586         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 587                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 588                         | DWC3_DEPCFG_STREAM_EVENT_EN
 589                         | DWC3_DEPCFG_XFER_COMPLETE_EN;
 590                 dep->stream_capable = true;
 591         }
 592
 593         if (!usb_endpoint_xfer_control(desc))
 594                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 595
 596         /*
 597          * We are doing 1:1 mapping for endpoints, meaning
 598          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 599          * so on. We consider the direction bit as part of the physical
 600          * endpoint number. So USB endpoint 0x81 is 0x03.
 601          */
 602         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 603
 604         /*
 605          * We must use the lower 16 TX FIFOs even though
 606          * HW might have more
 607          */
 608         if (dep->direction)
 609                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 610
 611         if (desc->bInterval) {
 612                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 613                 dep->interval = 1 << (desc->bInterval - 1);
 614         }
 615
 616         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 617 }
 618
 619 /**
 620  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 621  * @dep: endpoint to be initialized
 622  * @action: one of INIT, MODIFY or RESTORE
 623  *
 624  * Caller should take care of locking. Execute all necessary commands to
 625  * initialize a HW endpoint so it can be used by a gadget driver.
 626  */
 627 int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 628 {
 629         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 630         struct dwc3             *dwc = dep->dwc;
 631
 632         u32                     reg;
 633         int                     ret;
 634
 635         if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
 636                 ret = dwc3_gadget_start_config(dep);
 637                 if (ret)
 638                         return ret;
 639         }
 640
 641         ret = dwc3_gadget_set_ep_config(dep, action);
 642         if (ret)
 643                 return ret;
 644
 645         if (!(dep->flags & DWC3_EP_ENABLED) || dwc->is_hibernated) {
 646                 struct dwc3_trb *trb_st_hw;
 647                 struct dwc3_trb *trb_link;
 648
 649                 dep->type = usb_endpoint_type(desc);
 650                 dep->flags |= DWC3_EP_ENABLED;
 651                 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 652
 653                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 654                 reg |= DWC3_DALEPENA_EP(dep->number);
 655                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 656
 657                 init_waitqueue_head(&dep->wait_end_transfer);
 658
 659                 if (usb_endpoint_xfer_control(desc))
 660                         goto out;
 661
 662                 if (!dwc->is_hibernated) {
 663                         /* Initialize the TRB ring */
 664                         dep->trb_dequeue = 0;
 665                         dep->trb_enqueue = 0;
 666                         memset(dep->trb_pool, 0,
 667                                sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 668                 }
 669
 670                 /* Link TRB. The HWO bit is never reset */
 671                 trb_st_hw = &dep->trb_pool[0];
 672
 673                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 674                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 675                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 676                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 677                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 678         }
 679
 680         /*
 681          * Issue StartTransfer here with no-op TRB so we can always rely on No
 682          * Response Update Transfer command.
 683          */
 684         if (((usb_endpoint_xfer_bulk(desc) && !dep->stream_capable) ||
 685              usb_endpoint_xfer_int(desc)) && !dwc->is_hibernated) {
 686                 struct dwc3_gadget_ep_cmd_params params;
 687                 struct dwc3_trb *trb;
 688                 dma_addr_t trb_dma;
 689                 u32 cmd;
 690
 691                 memset(&params, 0, sizeof(params));
 692                 trb = &dep->trb_pool[0];
 693                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 694
 695                 params.param0 = upper_32_bits(trb_dma);
 696                 params.param1 = lower_32_bits(trb_dma);
 697
 698                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 699
 700                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 701                 if (ret < 0)
 702                         return ret;
 703         }
 704
 705 out:
 706         trace_dwc3_gadget_ep_enable(dep);
 707
 708         return 0;
 709 }
 710
 711 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 712 {
 713         struct dwc3_request             *req;
 714
 715         dwc3_stop_active_transfer(dep, true);
 716
 717         /* - giveback all requests to gadget driver */
 718         while (!list_empty(&dep->started_list)) {
 719                 req = next_request(&dep->started_list);
 720
 721                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 722         }
 723
 724         while (!list_empty(&dep->pending_list)) {
 725                 req = next_request(&dep->pending_list);
 726
 727                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 728         }
 729 }
 730
 731 /**
 732  * __dwc3_gadget_ep_disable - disables a hw endpoint
 733  * @dep: the endpoint to disable
 734  *
 735  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 736  * requests which are currently being processed by the hardware and those which
 737  * are not yet scheduled.
 738  *
 739  * Caller should take care of locking.
 740  */
 741 int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 742 {
 743         struct dwc3             *dwc = dep->dwc;
 744         u32                     reg;
 745
 746         trace_dwc3_gadget_ep_disable(dep);
 747
 748         dwc3_remove_requests(dwc, dep);
 749
 750         /* make sure HW endpoint isn't stalled */
 751         if (dep->flags & DWC3_EP_STALL)
 752                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 753
 754         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 755         reg &= ~DWC3_DALEPENA_EP(dep->number);
 756         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 757
 758         dep->stream_capable = false;
 759         dep->type = 0;
 760         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
 761
 762         /* Clear out the ep descriptors for non-ep0 */
 763         if (dep->number > 1) {
 764                 dep->endpoint.comp_desc = NULL;
 765                 dep->endpoint.desc = NULL;
 766         }
 767
 768         return 0;
 769 }
 770
 771 /* -------------------------------------------------------------------------- */
 772
 773 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 774                 const struct usb_endpoint_descriptor *desc)
 775 {
 776         return -EINVAL;
 777 }
 778
 779 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 780 {
 781         return -EINVAL;
 782 }
 783
 784 /* -------------------------------------------------------------------------- */
 785
 786 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 787                 const struct usb_endpoint_descriptor *desc)
 788 {
 789         struct dwc3_ep                  *dep;
 790         struct dwc3                     *dwc;
 791         unsigned long                   flags;
 792         int                             ret;
 793
 794         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 795                 pr_debug("dwc3: invalid parameters\n");
 796                 return -EINVAL;
 797         }
 798
 799         if (!desc->wMaxPacketSize) {
 800                 pr_debug("dwc3: missing wMaxPacketSize\n");
 801                 return -EINVAL;
 802         }
 803
 804         dep = to_dwc3_ep(ep);
 805         dwc = dep->dwc;
 806
 807         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 808                                         "%s is already enabled\n",
 809                                         dep->name))
 810                 return 0;
 811
 812         spin_lock_irqsave(&dwc->lock, flags);
 813         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 814         spin_unlock_irqrestore(&dwc->lock, flags);
 815
 816         return ret;
 817 }
 818
 819 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 820 {
 821         struct dwc3_ep                  *dep;
 822         struct dwc3                     *dwc;
 823         unsigned long                   flags;
 824         int                             ret;
 825
 826         if (!ep) {
 827                 pr_debug("dwc3: invalid parameters\n");
 828                 return -EINVAL;
 829         }
 830
 831         dep = to_dwc3_ep(ep);
 832         dwc = dep->dwc;
 833
 834         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 835                                         "%s is already disabled\n",
 836                                         dep->name))
 837                 return 0;
 838
 839         spin_lock_irqsave(&dwc->lock, flags);
 840         ret = __dwc3_gadget_ep_disable(dep);
 841         spin_unlock_irqrestore(&dwc->lock, flags);
 842
 843         return ret;
 844 }
 845
 846 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 847                 gfp_t gfp_flags)
 848 {
 849         struct dwc3_request             *req;
 850         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 851
 852         req = kzalloc(sizeof(*req), gfp_flags);
 853         if (!req)
 854                 return NULL;
 855
 856         req->direction  = dep->direction;
 857         req->epnum      = dep->number;
 858         req->dep        = dep;
 859
 860         trace_dwc3_alloc_request(req);
 861
 862         return &req->request;
 863 }
 864
 865 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 866                 struct usb_request *request)
 867 {
 868         struct dwc3_request             *req = to_dwc3_request(request);
 869
 870         trace_dwc3_free_request(req);
 871         kfree(req);
 872 }
 873
 874 /**
 875  * dwc3_ep_prev_trb - returns the previous TRB in the ring
 876  * @dep: The endpoint with the TRB ring
 877  * @index: The index of the current TRB in the ring
 878  *
 879  * Returns the TRB prior to the one pointed to by the index. If the
 880  * index is 0, we will wrap backwards, skip the link TRB, and return
 881  * the one just before that.
 882  */
 883 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
 884 {
 885         u8 tmp = index;
 886
 887         if (!tmp)
 888                 tmp = DWC3_TRB_NUM - 1;
 889
 890         return &dep->trb_pool[tmp - 1];
 891 }
 892
 893 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
 894 {
 895         struct dwc3_trb         *tmp;
 896         u8                      trbs_left;
 897
 898         /*
 899          * If enqueue & dequeue are equal than it is either full or empty.
 900          *
 901          * One way to know for sure is if the TRB right before us has HWO bit
 902          * set or not. If it has, then we're definitely full and can't fit any
 903          * more transfers in our ring.
 904          */
 905         if (dep->trb_enqueue == dep->trb_dequeue) {
 906                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
 907                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
 908                         return 0;
 909
 910                 return DWC3_TRB_NUM - 1;
 911         }
 912
 913         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
 914         trbs_left &= (DWC3_TRB_NUM - 1);
 915
 916         if (dep->trb_dequeue < dep->trb_enqueue)
 917                 trbs_left--;
 918
 919         return trbs_left;
 920 }
 921
 922 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 923                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 924                 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
 925 {
 926         struct dwc3             *dwc = dep->dwc;
 927         struct usb_gadget       *gadget = &dwc->gadget;
 928         enum usb_device_speed   speed = gadget->speed;
 929
 930         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 931         trb->bpl = lower_32_bits(dma);
 932         trb->bph = upper_32_bits(dma);
 933
 934         switch (usb_endpoint_type(dep->endpoint.desc)) {
 935         case USB_ENDPOINT_XFER_CONTROL:
 936                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 937                 break;
 938
 939         case USB_ENDPOINT_XFER_ISOC:
 940                 if (!node) {
 941                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 942
 943                         /*
 944                          * USB Specification 2.0 Section 5.9.2 states that: "If
 945                          * there is only a single transaction in the microframe,
 946                          * only a DATA0 data packet PID is used.  If there are
 947                          * two transactions per microframe, DATA1 is used for
 948                          * the first transaction data packet and DATA0 is used
 949                          * for the second transaction data packet.  If there are
 950                          * three transactions per microframe, DATA2 is used for
 951                          * the first transaction data packet, DATA1 is used for
 952                          * the second, and DATA0 is used for the third."
 953                          *
 954                          * IOW, we should satisfy the following cases:
 955                          *
 956                          * 1) length <= maxpacket
 957                          *      - DATA0
 958                          *
 959                          * 2) maxpacket < length <= (2 * maxpacket)
 960                          *      - DATA1, DATA0
 961                          *
 962                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 963                          *      - DATA2, DATA1, DATA0
 964                          */
 965                         if (speed == USB_SPEED_HIGH) {
 966                                 struct usb_ep *ep = &dep->endpoint;
 967                                 unsigned int mult = 2;
 968                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 969
 970                                 if (length <= (2 * maxp))
 971                                         mult--;
 972
 973                                 if (length <= maxp)
 974                                         mult--;
 975
 976                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
 977                         }
 978                 } else {
 979                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 980                 }
 981
 982                 /* always enable Interrupt on Missed ISOC */
 983                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 984                 break;
 985
 986         case USB_ENDPOINT_XFER_BULK:
 987         case USB_ENDPOINT_XFER_INT:
 988                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 989                 break;
 990         default:
 991                 /*
 992                  * This is only possible with faulty memory because we
 993                  * checked it already :)
 994                  */
 995                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
 996                                 usb_endpoint_type(dep->endpoint.desc));
 997         }
 998
 999         /* always enable Continue on Short Packet */
1000         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
1001                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
1002
1003                 if (short_not_ok)
1004                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
1005         }
1006
1007         if ((!no_interrupt && !chain) ||
1008             (dwc3_calc_trbs_left(dep) == 1))
1009                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
1010
1011         if (chain)
1012                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
1013         /*
1014          * To start transfer on another stream number endpoint need to relase
1015          * previously acquired transfer resource for doing that there is two
1016          * ways 1. end transfer 2. set lst bit of control trb
1017          *
1018          * by using lst bit in ctrl trb we will be able to save the time of
1019          * ending transfer hence improved performance
1020          */
1021         else if (dep->stream_capable)
1022                 trb->ctrl |= DWC3_TRB_CTRL_LST;
1023
1024         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
1025                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
1026
1027         trb->ctrl |= DWC3_TRB_CTRL_HWO;
1028
1029         dwc3_ep_inc_enq(dep);
1030
1031         trace_dwc3_prepare_trb(dep, trb);
1032 }
1033
1034 /**
1035  * dwc3_prepare_one_trb - setup one TRB from one request
1036  * @dep: endpoint for which this request is prepared
1037  * @req: dwc3_request pointer
1038  * @chain: should this TRB be chained to the next?
1039  * @node: only for isochronous endpoints. First TRB needs different type.
1040  */
1041 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
1042                 struct dwc3_request *req, unsigned chain, unsigned node)
1043 {
1044         struct dwc3_trb         *trb;
1045         unsigned int            length;
1046         dma_addr_t              dma;
1047         unsigned                stream_id = req->request.stream_id;
1048         unsigned                short_not_ok = req->request.short_not_ok;
1049         unsigned                no_interrupt = req->request.no_interrupt;
1050
1051         if (req->request.num_sgs > 0) {
1052                 length = sg_dma_len(req->start_sg);
1053                 dma = sg_dma_address(req->start_sg);
1054         } else {
1055                 length = req->request.length;
1056                 dma = req->request.dma;
1057         }
1058
1059         trb = &dep->trb_pool[dep->trb_enqueue];
1060
1061         if (!req->trb) {
1062                 dwc3_gadget_move_started_request(req);
1063                 req->trb = trb;
1064                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
1065         }
1066
1067         __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
1068                         stream_id, short_not_ok, no_interrupt);
1069 }
1070
1071 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1072                 struct dwc3_request *req)
1073 {
1074         struct scatterlist *sg = req->start_sg;
1075         struct scatterlist *s;
1076         int             i;
1077
1078         unsigned int remaining = req->request.num_mapped_sgs
1079                 - req->num_queued_sgs;
1080
1081         for_each_sg(sg, s, remaining, i) {
1082                 unsigned int length = req->request.length;
1083                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1084                 unsigned int rem = length % maxp;
1085                 unsigned chain = true;
1086
1087                 if (sg_is_last(s))
1088                         chain = false;
1089
1090                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1091                         struct dwc3     *dwc = dep->dwc;
1092                         struct dwc3_trb *trb;
1093
1094                         req->unaligned = true;
1095
1096                         /* prepare normal TRB */
1097                         dwc3_prepare_one_trb(dep, req, true, i);
1098
1099                         /* Now prepare one extra TRB to align transfer size */
1100                         trb = &dep->trb_pool[dep->trb_enqueue];
1101                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1102                                         maxp - rem, false, 0,
1103                                         req->request.stream_id,
1104                                         req->request.short_not_ok,
1105                                         req->request.no_interrupt);
1106                 } else {
1107                         dwc3_prepare_one_trb(dep, req, chain, i);
1108                 }
1109
1110                 /*
1111                  * There can be a situation where all sgs in sglist are not
1112                  * queued because of insufficient trb number. To handle this
1113                  * case, update start_sg to next sg to be queued, so that
1114                  * we have free trbs we can continue queuing from where we
1115                  * previously stopped
1116                  */
1117                 if (chain)
1118                         req->start_sg = sg_next(s);
1119
1120                 req->num_queued_sgs++;
1121
1122                 if (!dwc3_calc_trbs_left(dep))
1123                         break;
1124         }
1125 }
1126
1127 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1128                 struct dwc3_request *req)
1129 {
1130         unsigned int length = req->request.length;
1131         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1132         unsigned int rem = length % maxp;
1133
1134         if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1135                 struct dwc3     *dwc = dep->dwc;
1136                 struct dwc3_trb *trb;
1137
1138                 req->unaligned = true;
1139
1140                 /* prepare normal TRB */
1141                 dwc3_prepare_one_trb(dep, req, true, 0);
1142
1143                 /* Now prepare one extra TRB to align transfer size */
1144                 trb = &dep->trb_pool[dep->trb_enqueue];
1145                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1146                                 false, 0, req->request.stream_id,
1147                                 req->request.short_not_ok,
1148                                 req->request.no_interrupt);
1149         } else if (req->request.zero && req->request.length &&
1150                    (IS_ALIGNED(req->request.length, maxp))) {
1151                 struct dwc3     *dwc = dep->dwc;
1152                 struct dwc3_trb *trb;
1153
1154                 req->zero = true;
1155
1156                 /* prepare normal TRB */
1157                 dwc3_prepare_one_trb(dep, req, true, 0);
1158
1159                 /* Now prepare one extra TRB to handle ZLP */
1160                 trb = &dep->trb_pool[dep->trb_enqueue];
1161                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1162                                 false, 0, req->request.stream_id,
1163                                 req->request.short_not_ok,
1164                                 req->request.no_interrupt);
1165         } else {
1166                 dwc3_prepare_one_trb(dep, req, false, 0);
1167         }
1168 }
1169
1170 /*
1171  * dwc3_prepare_trbs - setup TRBs from requests
1172  * @dep: endpoint for which requests are being prepared
1173  *
1174  * The function goes through the requests list and sets up TRBs for the
1175  * transfers. The function returns once there are no more TRBs available or
1176  * it runs out of requests.
1177  */
1178 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1179 {
1180         struct dwc3_request     *req, *n;
1181
1182         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1183
1184         /*
1185          * We can get in a situation where there's a request in the started list
1186          * but there weren't enough TRBs to fully kick it in the first time
1187          * around, so it has been waiting for more TRBs to be freed up.
1188          *
1189          * In that case, we should check if we have a request with pending_sgs
1190          * in the started list and prepare TRBs for that request first,
1191          * otherwise we will prepare TRBs completely out of order and that will
1192          * break things.
1193          */
1194         list_for_each_entry(req, &dep->started_list, list) {
1195                 if (req->num_pending_sgs > 0)
1196                         dwc3_prepare_one_trb_sg(dep, req);
1197
1198                 if (!dwc3_calc_trbs_left(dep))
1199                         return;
1200         }
1201
1202         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1203                 struct dwc3     *dwc = dep->dwc;
1204                 int             ret;
1205
1206                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1207                                                     dep->direction);
1208                 if (ret)
1209                         return;
1210
1211                 req->sg                 = req->request.sg;
1212                 req->start_sg           = req->sg;
1213                 req->num_queued_sgs     = 0;
1214                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1215
1216                 if (req->num_pending_sgs > 0)
1217                         dwc3_prepare_one_trb_sg(dep, req);
1218                 else
1219                         dwc3_prepare_one_trb_linear(dep, req);
1220
1221                 if (!dwc3_calc_trbs_left(dep))
1222                         return;
1223         }
1224 }
1225
1226 int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1227 {
1228         struct dwc3_gadget_ep_cmd_params params;
1229         struct dwc3_request             *req;
1230         int                             starting;
1231         int                             ret;
1232         u32                             cmd;
1233
1234         if (!dwc3_calc_trbs_left(dep))
1235                 return 0;
1236
1237         starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1238
1239         dwc3_prepare_trbs(dep);
1240         req = next_request(&dep->started_list);
1241         if (!req) {
1242                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1243                 return 0;
1244         }
1245
1246         memset(&params, 0, sizeof(params));
1247
1248         if (starting) {
1249                 params.param0 = upper_32_bits(req->trb_dma);
1250                 params.param1 = lower_32_bits(req->trb_dma);
1251                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1252
1253                 if (dep->stream_capable)
1254                         cmd = cmd | DWC3_DEPCMD_PARAM(req->request.stream_id);
1255
1256                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1257                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1258
1259         } else {
1260                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1261                         DWC3_DEPCMD_PARAM(dep->resource_index);
1262         }
1263
1264         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1265         if (ret < 0) {
1266                 /*
1267                  * FIXME we need to iterate over the list of requests
1268                  * here and stop, unmap, free and del each of the linked
1269                  * requests instead of what we do now.
1270                  */
1271                 if (req->trb)
1272                         memset(req->trb, 0, sizeof(struct dwc3_trb));
1273                 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1274                 return ret;
1275         }
1276
1277         if (starting && dep->stream_capable) {
1278                 req->stream_timeout_timer.expires = jiffies +
1279                                 msecs_to_jiffies(STREAM_TIMEOUT_MS);
1280                 mod_timer(&req->stream_timeout_timer,
1281                           req->stream_timeout_timer.expires);
1282         }
1283
1284         return 0;
1285 }
1286
1287 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1288 {
1289         u32                     reg;
1290
1291         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1292         return DWC3_DSTS_SOFFN(reg);
1293 }
1294
1295 static void __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1296 {
1297         if (list_empty(&dep->pending_list)) {
1298                 dev_info(dep->dwc->dev, "%s: ran out of requests\n",
1299                                 dep->name);
1300                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1301                 return;
1302         }
1303
1304         dep->frame_number = DWC3_ALIGN_FRAME(dep);
1305         __dwc3_gadget_kick_transfer(dep);
1306 }
1307
1308 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc);
1309 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1310 {
1311         struct dwc3             *dwc = dep->dwc;
1312
1313         if (!dep->endpoint.desc) {
1314                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1315                                 dep->name);
1316                 return -ESHUTDOWN;
1317         }
1318
1319         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1320                                 &req->request, req->dep->name))
1321                 return -EINVAL;
1322
1323         pm_runtime_get(dwc->dev);
1324
1325         req->request.actual     = 0;
1326         req->request.status     = -EINPROGRESS;
1327
1328         if (dep->stream_capable)
1329                 timer_setup(&req->stream_timeout_timer,
1330                             stream_timeout_function, 0);
1331
1332         trace_dwc3_ep_queue(req);
1333
1334         list_add_tail(&req->list, &dep->pending_list);
1335
1336         /* If core is hibernated, need to wakeup (remote wakeup) */
1337         if (dwc->is_hibernated) {
1338                 dwc->force_hiber_wake = true;
1339                 gadget_wakeup_interrupt(dwc);
1340                 dwc->force_hiber_wake = false;
1341         }
1342
1343         /*
1344          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1345          * wait for a XferNotReady event so we will know what's the current
1346          * (micro-)frame number.
1347          *
1348          * Without this trick, we are very, very likely gonna get Bus Expiry
1349          * errors which will force us issue EndTransfer command.
1350          */
1351         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1352                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1353                         if (dep->flags & DWC3_EP_TRANSFER_STARTED) {
1354                                 dwc3_stop_active_transfer(dep, true);
1355                                 dep->flags = DWC3_EP_ENABLED;
1356                         } else {
1357                                 u32 cur_uf;
1358
1359                                 cur_uf = __dwc3_gadget_get_frame(dwc);
1360                                 __dwc3_gadget_start_isoc(dep);
1361                                 dep->flags &= ~DWC3_EP_PENDING_REQUEST;
1362                         }
1363                         return 0;
1364                 }
1365
1366                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1367                         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1368                                 __dwc3_gadget_start_isoc(dep);
1369                                 return 0;
1370                         }
1371                 }
1372         }
1373
1374         return __dwc3_gadget_kick_transfer(dep);
1375 }
1376
1377 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1378         gfp_t gfp_flags)
1379 {
1380         struct dwc3_request             *req = to_dwc3_request(request);
1381         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1382         struct dwc3                     *dwc = dep->dwc;
1383
1384         unsigned long                   flags;
1385
1386         int                             ret;
1387
1388         spin_lock_irqsave(&dwc->lock, flags);
1389         ret = __dwc3_gadget_ep_queue(dep, req);
1390         spin_unlock_irqrestore(&dwc->lock, flags);
1391
1392         return ret;
1393 }
1394
1395 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1396                 struct usb_request *request)
1397 {
1398         struct dwc3_request             *req = to_dwc3_request(request);
1399         struct dwc3_request             *r = NULL;
1400
1401         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1402         struct dwc3                     *dwc = dep->dwc;
1403
1404         unsigned long                   flags;
1405         int                             ret = 0;
1406
1407         trace_dwc3_ep_dequeue(req);
1408
1409         spin_lock_irqsave(&dwc->lock, flags);
1410
1411         if (dep->stream_capable && timer_pending(&req->stream_timeout_timer))
1412                 del_timer(&req->stream_timeout_timer);
1413
1414         list_for_each_entry(r, &dep->pending_list, list) {
1415                 if (r == req)
1416                         break;
1417         }
1418
1419         if (r != req) {
1420                 list_for_each_entry(r, &dep->started_list, list) {
1421                         if (r == req)
1422                                 break;
1423                 }
1424                 if (r == req) {
1425                         /* wait until it is processed */
1426                         dwc3_stop_active_transfer(dep, true);
1427
1428                         /*
1429                          * If request was already started, this means we had to
1430                          * stop the transfer. With that we also need to ignore
1431                          * all TRBs used by the request, however TRBs can only
1432                          * be modified after completion of END_TRANSFER
1433                          * command. So what we do here is that we wait for
1434                          * END_TRANSFER completion and only after that, we jump
1435                          * over TRBs by clearing HWO and incrementing dequeue
1436                          * pointer.
1437                          *
1438                          * Note that we have 2 possible types of transfers here:
1439                          *
1440                          * i) Linear buffer request
1441                          * ii) SG-list based request
1442                          *
1443                          * SG-list based requests will have r->num_pending_sgs
1444                          * set to a valid number (> 0). Linear requests,
1445                          * normally use a single TRB.
1446                          *
1447                          * For each of these two cases, if r->unaligned flag is
1448                          * set, one extra TRB has been used to align transfer
1449                          * size to wMaxPacketSize.
1450                          *
1451                          * All of these cases need to be taken into
1452                          * consideration so we don't mess up our TRB ring
1453                          * pointers.
1454                          */
1455                         wait_event_lock_irq(dep->wait_end_transfer,
1456                                         !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1457                                         dwc->lock);
1458
1459                         if (!r->trb)
1460                                 goto out0;
1461
1462                         if (r->num_pending_sgs) {
1463                                 struct dwc3_trb *trb;
1464                                 int i = 0;
1465
1466                                 for (i = 0; i < r->num_pending_sgs; i++) {
1467                                         trb = r->trb + i;
1468                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1469                                         dwc3_ep_inc_deq(dep);
1470                                 }
1471
1472                                 if (r->unaligned || r->zero) {
1473                                         trb = r->trb + r->num_pending_sgs + 1;
1474                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1475                                         dwc3_ep_inc_deq(dep);
1476                                 }
1477                         } else {
1478                                 struct dwc3_trb *trb = r->trb;
1479
1480                                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1481                                 dwc3_ep_inc_deq(dep);
1482
1483                                 if (r->unaligned || r->zero) {
1484                                         trb = r->trb + 1;
1485                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1486                                         dwc3_ep_inc_deq(dep);
1487                                 }
1488                         }
1489                         goto out1;
1490                 }
1491                 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1492                                 request, ep->name);
1493                 ret = -EINVAL;
1494                 goto out0;
1495         }
1496
1497 out1:
1498         /* giveback the request */
1499
1500         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1501
1502 out0:
1503         spin_unlock_irqrestore(&dwc->lock, flags);
1504
1505         return ret;
1506 }
1507
1508 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1509 {
1510         struct dwc3_gadget_ep_cmd_params        params;
1511         struct dwc3                             *dwc = dep->dwc;
1512         int                                     ret;
1513
1514         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1515                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1516                 return -EINVAL;
1517         }
1518
1519         memset(&params, 0x00, sizeof(params));
1520
1521         if (value) {
1522                 struct dwc3_trb *trb;
1523
1524                 unsigned transfer_in_flight;
1525                 unsigned started;
1526
1527                 if (dep->flags & DWC3_EP_STALL)
1528                         return 0;
1529
1530                 if (dep->number > 1)
1531                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1532                 else
1533                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1534
1535                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1536                 started = !list_empty(&dep->started_list);
1537
1538                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1539                                 (!dep->direction && started))) {
1540                         return -EAGAIN;
1541                 }
1542
1543                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1544                                 &params);
1545                 if (ret)
1546                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1547                                         dep->name);
1548                 else
1549                         dep->flags |= DWC3_EP_STALL;
1550         } else {
1551                 if (!(dep->flags & DWC3_EP_STALL))
1552                         return 0;
1553
1554                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1555                 if (ret)
1556                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1557                                         dep->name);
1558                 else
1559                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1560         }
1561
1562         return ret;
1563 }
1564
1565 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1566 {
1567         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1568         struct dwc3                     *dwc = dep->dwc;
1569
1570         unsigned long                   flags;
1571
1572         int                             ret;
1573
1574         spin_lock_irqsave(&dwc->lock, flags);
1575         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1576         spin_unlock_irqrestore(&dwc->lock, flags);
1577
1578         return ret;
1579 }
1580
1581 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1582 {
1583         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1584         struct dwc3                     *dwc = dep->dwc;
1585         unsigned long                   flags;
1586         int                             ret;
1587
1588         spin_lock_irqsave(&dwc->lock, flags);
1589         dep->flags |= DWC3_EP_WEDGE;
1590
1591         if (dep->number == 0 || dep->number == 1)
1592                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1593         else
1594                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1595         spin_unlock_irqrestore(&dwc->lock, flags);
1596
1597         return ret;
1598 }
1599
1600 /* -------------------------------------------------------------------------- */
1601
1602 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1603         .bLength        = USB_DT_ENDPOINT_SIZE,
1604         .bDescriptorType = USB_DT_ENDPOINT,
1605         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1606 };
1607
1608 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1609         .enable         = dwc3_gadget_ep0_enable,
1610         .disable        = dwc3_gadget_ep0_disable,
1611         .alloc_request  = dwc3_gadget_ep_alloc_request,
1612         .free_request   = dwc3_gadget_ep_free_request,
1613         .queue          = dwc3_gadget_ep0_queue,
1614         .dequeue        = dwc3_gadget_ep_dequeue,
1615         .set_halt       = dwc3_gadget_ep0_set_halt,
1616         .set_wedge      = dwc3_gadget_ep_set_wedge,
1617 };
1618
1619 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1620         .enable         = dwc3_gadget_ep_enable,
1621         .disable        = dwc3_gadget_ep_disable,
1622         .alloc_request  = dwc3_gadget_ep_alloc_request,
1623         .free_request   = dwc3_gadget_ep_free_request,
1624         .queue          = dwc3_gadget_ep_queue,
1625         .dequeue        = dwc3_gadget_ep_dequeue,
1626         .set_halt       = dwc3_gadget_ep_set_halt,
1627         .set_wedge      = dwc3_gadget_ep_set_wedge,
1628 };
1629
1630 /* -------------------------------------------------------------------------- */
1631
1632 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1633 {
1634         struct dwc3             *dwc = gadget_to_dwc(g);
1635
1636         return __dwc3_gadget_get_frame(dwc);
1637 }
1638
1639 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1640 {
1641         int                     retries;
1642
1643         int                     ret;
1644         u32                     reg;
1645
1646         u8                      link_state;
1647         u8                      speed;
1648
1649         /*
1650          * According to the Databook Remote wakeup request should
1651          * be issued only when the device is in early suspend state.
1652          *
1653          * We can check that via USB Link State bits in DSTS register.
1654          */
1655         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1656
1657         speed = reg & DWC3_DSTS_CONNECTSPD;
1658         if ((speed == DWC3_DSTS_SUPERSPEED) ||
1659             (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1660                 return 0;
1661
1662         link_state = DWC3_DSTS_USBLNKST(reg);
1663
1664         switch (link_state) {
1665         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1666         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1667                 break;
1668         default:
1669                 return -EINVAL;
1670         }
1671
1672         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1673         if (ret < 0) {
1674                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1675                 return ret;
1676         }
1677
1678         /* Recent versions do this automatically */
1679         if (dwc->revision < DWC3_REVISION_194A) {
1680                 /* write zeroes to Link Change Request */
1681                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1682                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1683                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1684         }
1685
1686         /* poll until Link State changes to ON */
1687         retries = 20000;
1688
1689         while (retries--) {
1690                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1691
1692                 /* in HS, means ON */
1693                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1694                         break;
1695         }
1696
1697         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1698                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1699                 return -EINVAL;
1700         }
1701
1702         return 0;
1703 }
1704
1705 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1706 {
1707         struct dwc3             *dwc = gadget_to_dwc(g);
1708         unsigned long           flags;
1709         int                     ret;
1710
1711         spin_lock_irqsave(&dwc->lock, flags);
1712         ret = __dwc3_gadget_wakeup(dwc);
1713         spin_unlock_irqrestore(&dwc->lock, flags);
1714
1715         return ret;
1716 }
1717
1718 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1719                 int is_selfpowered)
1720 {
1721         struct dwc3             *dwc = gadget_to_dwc(g);
1722         unsigned long           flags;
1723
1724         spin_lock_irqsave(&dwc->lock, flags);
1725         g->is_selfpowered = !!is_selfpowered;
1726         spin_unlock_irqrestore(&dwc->lock, flags);
1727
1728         return 0;
1729 }
1730
1731 int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1732 {
1733         u32                     reg;
1734         u32                     timeout = 500;
1735
1736         if (pm_runtime_suspended(dwc->dev))
1737                 return 0;
1738
1739         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1740         if (is_on) {
1741                 if (dwc->revision <= DWC3_REVISION_187A) {
1742                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1743                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1744                 }
1745
1746                 if (dwc->revision >= DWC3_REVISION_194A)
1747                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1748                 reg |= DWC3_DCTL_RUN_STOP;
1749
1750                 if (dwc->has_hibernation)
1751                         reg |= DWC3_DCTL_KEEP_CONNECT;
1752
1753                 dwc->pullups_connected = true;
1754         } else {
1755                 reg &= ~DWC3_DCTL_RUN_STOP;
1756
1757                 if (dwc->has_hibernation && !suspend)
1758                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1759
1760                 dwc->pullups_connected = false;
1761         }
1762
1763         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1764
1765         do {
1766                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1767                 reg &= DWC3_DSTS_DEVCTRLHLT;
1768         } while (--timeout && !(!is_on ^ !reg));
1769
1770         if (!timeout)
1771                 return -ETIMEDOUT;
1772
1773         return 0;
1774 }
1775
1776 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1777 {
1778         struct dwc3             *dwc = gadget_to_dwc(g);
1779         unsigned long           flags;
1780         int                     ret;
1781
1782         is_on = !!is_on;
1783
1784         /*
1785          * Per databook, when we want to stop the gadget, if a control transfer
1786          * is still in process, complete it and get the core into setup phase.
1787          */
1788         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1789                 reinit_completion(&dwc->ep0_in_setup);
1790
1791                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1792                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1793                 if (ret == 0) {
1794                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1795                         return -ETIMEDOUT;
1796                 }
1797         }
1798
1799         spin_lock_irqsave(&dwc->lock, flags);
1800         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1801         spin_unlock_irqrestore(&dwc->lock, flags);
1802
1803         return ret;
1804 }
1805
1806 void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1807 {
1808         u32                     reg;
1809
1810         /* Enable all but Start and End of Frame IRQs */
1811         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1812                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1813                         DWC3_DEVTEN_CMDCMPLTEN |
1814                         DWC3_DEVTEN_ERRTICERREN |
1815                         DWC3_DEVTEN_WKUPEVTEN |
1816                         DWC3_DEVTEN_CONNECTDONEEN |
1817                         DWC3_DEVTEN_USBRSTEN |
1818                         DWC3_DEVTEN_DISCONNEVTEN);
1819
1820         /* Enable hibernation IRQ */
1821         if (dwc->has_hibernation)
1822                 reg |= DWC3_DEVTEN_HIBERNATIONREQEVTEN;
1823
1824         if (dwc->revision < DWC3_REVISION_250A)
1825                 reg |= DWC3_DEVTEN_ULSTCNGEN;
1826
1827         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1828 }
1829
1830 void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1831 {
1832         /* mask all interrupts */
1833         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1834 }
1835
1836 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1837 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1838
1839 /**
1840  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1841  * @dwc: pointer to our context structure
1842  *
1843  * The following looks like complex but it's actually very simple. In order to
1844  * calculate the number of packets we can burst at once on OUT transfers, we're
1845  * gonna use RxFIFO size.
1846  *
1847  * To calculate RxFIFO size we need two numbers:
1848  * MDWIDTH = size, in bits, of the internal memory bus
1849  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1850  *
1851  * Given these two numbers, the formula is simple:
1852  *
1853  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1854  *
1855  * 24 bytes is for 3x SETUP packets
1856  * 16 bytes is a clock domain crossing tolerance
1857  *
1858  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1859  */
1860 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1861 {
1862         u32 ram2_depth;
1863         u32 mdwidth;
1864         u32 nump;
1865         u32 reg;
1866
1867         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1868         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1869
1870         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1871         nump = min_t(u32, nump, 16);
1872
1873         /* update NumP */
1874         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1875         reg &= ~DWC3_DCFG_NUMP_MASK;
1876         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1877         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1878 }
1879
1880 static int __dwc3_gadget_start(struct dwc3 *dwc)
1881 {
1882         struct dwc3_ep          *dep;
1883         int                     ret = 0;
1884         u32                     reg;
1885
1886         /*
1887          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1888          * the core supports IMOD, disable it.
1889          */
1890         if (dwc->imod_interval) {
1891                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1892                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1893         } else if (dwc3_has_imod(dwc)) {
1894                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1895         }
1896
1897         /*
1898          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1899          * field instead of letting dwc3 itself calculate that automatically.
1900          *
1901          * This way, we maximize the chances that we'll be able to get several
1902          * bursts of data without going through any sort of endpoint throttling.
1903          */
1904         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1905         if (dwc3_is_usb31(dwc))
1906                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1907         else
1908                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1909
1910         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1911
1912         dwc3_gadget_setup_nump(dwc);
1913
1914         /* For OTG mode, check if the core is currently in Host mode.
1915          * This is not an error condition as there are times when the core is
1916          * working as host and kernel is told to initiate bind operation with
1917          * gadget class driver module.
1918          * The below remaining operations are handled in OTG driver whenever
1919          * required.
1920          */
1921         if (dwc3_readl(dwc->regs, DWC3_GSTS) & DWC3_GSTS_CUR_MODE)
1922                 return 0;
1923
1924         /* Start with SuperSpeed Default */
1925         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1926
1927         dep = dwc->eps[0];
1928         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1929         if (ret) {
1930                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1931                 goto err0;
1932         }
1933
1934         dep = dwc->eps[1];
1935         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1936         if (ret) {
1937                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1938                 goto err1;
1939         }
1940
1941         /* begin to receive SETUP packets */
1942         dwc->ep0state = EP0_SETUP_PHASE;
1943         dwc3_ep0_out_start(dwc);
1944
1945         dwc3_gadget_enable_irq(dwc);
1946
1947         return 0;
1948
1949 err1:
1950         __dwc3_gadget_ep_disable(dwc->eps[0]);
1951
1952 err0:
1953         return ret;
1954 }
1955
1956 static irqreturn_t wakeup_interrupt(int irq, void *_dwc);
1957 static int dwc3_gadget_start(struct usb_gadget *g,
1958                 struct usb_gadget_driver *driver)
1959 {
1960         struct dwc3             *dwc = gadget_to_dwc(g);
1961         unsigned long           flags;
1962         int                     ret = 0;
1963         int                     irq;
1964
1965         irq = dwc->irq_gadget;
1966         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1967                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1968         if (ret) {
1969                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1970                                 irq, ret);
1971                 goto err0;
1972         }
1973
1974         /* look for wakeup interrupt if hibernation is supported */
1975         if (dwc->has_hibernation) {
1976                 irq = dwc->irq_wakeup;
1977                 ret = devm_request_irq(dwc->dev, irq, wakeup_interrupt,
1978                                        IRQF_SHARED, "usb-wakeup", dwc);
1979                 if (ret) {
1980                         dev_err(dwc->dev, "failed to request wakeup irq #%d --> %d\n",
1981                                 irq, ret);
1982                         goto err0;
1983                 }
1984         }
1985
1986         spin_lock_irqsave(&dwc->lock, flags);
1987         if (dwc->gadget_driver) {
1988                 dev_err(dwc->dev, "%s is already bound to %s\n",
1989                                 dwc->gadget.name,
1990                                 dwc->gadget_driver->driver.name);
1991                 ret = -EBUSY;
1992                 goto err1;
1993         }
1994
1995         dwc->gadget_driver      = driver;
1996
1997         if (pm_runtime_active(dwc->dev))
1998                 __dwc3_gadget_start(dwc);
1999
2000         spin_unlock_irqrestore(&dwc->lock, flags);
2001
2002         return 0;
2003
2004 err1:
2005         spin_unlock_irqrestore(&dwc->lock, flags);
2006         if (dwc->irq_gadget)
2007                 free_irq(dwc->irq_gadget, dwc->ev_buf);
2008         if (dwc->irq_wakeup)
2009                 free_irq(dwc->irq_wakeup, dwc);
2010
2011 err0:
2012         return ret;
2013 }
2014
2015 static void __dwc3_gadget_stop(struct dwc3 *dwc)
2016 {
2017         dwc3_gadget_disable_irq(dwc);
2018         __dwc3_gadget_ep_disable(dwc->eps[0]);
2019         __dwc3_gadget_ep_disable(dwc->eps[1]);
2020 }
2021
2022 static int dwc3_gadget_stop(struct usb_gadget *g)
2023 {
2024         struct dwc3             *dwc = gadget_to_dwc(g);
2025         unsigned long           flags;
2026         int                     epnum;
2027         u32                     tmo_eps = 0;
2028
2029         spin_lock_irqsave(&dwc->lock, flags);
2030
2031         if (pm_runtime_suspended(dwc->dev))
2032                 goto out;
2033
2034         __dwc3_gadget_stop(dwc);
2035
2036         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2037                 struct dwc3_ep  *dep = dwc->eps[epnum];
2038                 int ret;
2039
2040                 if (!dep)
2041                         continue;
2042
2043                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2044                         continue;
2045
2046                 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer,
2047                             !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
2048                             dwc->lock, msecs_to_jiffies(5));
2049
2050                 if (ret <= 0) {
2051                         /* Timed out or interrupted! There's nothing much
2052                          * we can do so we just log here and print which
2053                          * endpoints timed out at the end.
2054                          */
2055                         tmo_eps |= 1 << epnum;
2056                         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
2057                 }
2058         }
2059
2060         if (tmo_eps) {
2061                 dev_err(dwc->dev,
2062                         "end transfer timed out on endpoints 0x%x [bitmap]\n",
2063                         tmo_eps);
2064         }
2065
2066 out:
2067         dwc->gadget_driver      = NULL;
2068         spin_unlock_irqrestore(&dwc->lock, flags);
2069
2070         free_irq(dwc->irq_gadget, dwc->ev_buf);
2071         free_irq(dwc->irq_wakeup, dwc);
2072
2073         return 0;
2074 }
2075
2076 static void dwc3_gadget_set_speed(struct usb_gadget *g,
2077                                   enum usb_device_speed speed)
2078 {
2079         struct dwc3             *dwc = gadget_to_dwc(g);
2080         unsigned long           flags;
2081         u32                     reg;
2082
2083         spin_lock_irqsave(&dwc->lock, flags);
2084         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2085         reg &= ~(DWC3_DCFG_SPEED_MASK);
2086
2087         /*
2088          * WORKAROUND: DWC3 revision < 2.20a have an issue
2089          * which would cause metastability state on Run/Stop
2090          * bit if we try to force the IP to USB2-only mode.
2091          *
2092          * Because of that, we cannot configure the IP to any
2093          * speed other than the SuperSpeed
2094          *
2095          * Refers to:
2096          *
2097          * STAR#9000525659: Clock Domain Crossing on DCTL in
2098          * USB 2.0 Mode
2099          */
2100         if (dwc->revision < DWC3_REVISION_220A &&
2101             !dwc->dis_metastability_quirk) {
2102                 reg |= DWC3_DCFG_SUPERSPEED;
2103         } else {
2104                 switch (speed) {
2105                 case USB_SPEED_LOW:
2106                         reg |= DWC3_DCFG_LOWSPEED;
2107                         break;
2108                 case USB_SPEED_FULL:
2109                         reg |= DWC3_DCFG_FULLSPEED;
2110                         break;
2111                 case USB_SPEED_HIGH:
2112                         reg |= DWC3_DCFG_HIGHSPEED;
2113                         break;
2114                 case USB_SPEED_SUPER:
2115                         reg |= DWC3_DCFG_SUPERSPEED;
2116                         break;
2117                 case USB_SPEED_SUPER_PLUS:
2118                         if (dwc3_is_usb31(dwc))
2119                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2120                         else
2121                                 reg |= DWC3_DCFG_SUPERSPEED;
2122                         break;
2123                 default:
2124                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2125
2126                         if (dwc->revision & DWC3_REVISION_IS_DWC31)
2127                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2128                         else
2129                                 reg |= DWC3_DCFG_SUPERSPEED;
2130                 }
2131         }
2132         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2133
2134         spin_unlock_irqrestore(&dwc->lock, flags);
2135 }
2136
2137 static const struct usb_gadget_ops dwc3_gadget_ops = {
2138         .get_frame              = dwc3_gadget_get_frame,
2139         .wakeup                 = dwc3_gadget_wakeup,
2140         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2141         .pullup                 = dwc3_gadget_pullup,
2142         .udc_start              = dwc3_gadget_start,
2143         .udc_stop               = dwc3_gadget_stop,
2144         .udc_set_speed          = dwc3_gadget_set_speed,
2145 };
2146
2147 /* -------------------------------------------------------------------------- */
2148
2149 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2150 {
2151         struct dwc3 *dwc = dep->dwc;
2152
2153         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2154         dep->endpoint.maxburst = 1;
2155         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2156         if (!dep->direction)
2157                 dwc->gadget.ep0 = &dep->endpoint;
2158
2159         dep->endpoint.caps.type_control = true;
2160
2161         return 0;
2162 }
2163
2164 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2165 {
2166         struct dwc3 *dwc = dep->dwc;
2167         int mdwidth;
2168         int kbytes;
2169         int size;
2170
2171         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2172         /* MDWIDTH is represented in bits, we need it in bytes */
2173         mdwidth /= 8;
2174
2175         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2176         if (dwc3_is_usb31(dwc))
2177                 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2178         else
2179                 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2180
2181         /* FIFO Depth is in MDWDITH bytes. Multiply */
2182         size *= mdwidth;
2183
2184         kbytes = size / 1024;
2185         if (kbytes == 0)
2186                 kbytes = 1;
2187
2188         /*
2189          * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
2190          * internal overhead. We don't really know how these are used,
2191          * but documentation say it exists.
2192          */
2193         size -= mdwidth * (kbytes + 1);
2194         size /= kbytes;
2195
2196         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2197
2198         dep->endpoint.max_streams = 15;
2199         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2200         list_add_tail(&dep->endpoint.ep_list,
2201                         &dwc->gadget.ep_list);
2202         dep->endpoint.caps.type_iso = true;
2203         dep->endpoint.caps.type_bulk = true;
2204         dep->endpoint.caps.type_int = true;
2205
2206         return dwc3_alloc_trb_pool(dep);
2207 }
2208
2209 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2210 {
2211         struct dwc3 *dwc = dep->dwc;
2212
2213         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2214         dep->endpoint.max_streams = 15;
2215         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2216         list_add_tail(&dep->endpoint.ep_list,
2217                         &dwc->gadget.ep_list);
2218         dep->endpoint.caps.type_iso = true;
2219         dep->endpoint.caps.type_bulk = true;
2220         dep->endpoint.caps.type_int = true;
2221
2222         return dwc3_alloc_trb_pool(dep);
2223 }
2224
2225 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2226 {
2227         struct dwc3_ep                  *dep;
2228         bool                            direction = epnum & 1;
2229         int                             ret;
2230         u8                              num = epnum >> 1;
2231
2232         dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2233         if (!dep)
2234                 return -ENOMEM;
2235
2236         dep->dwc = dwc;
2237         dep->number = epnum;
2238         dep->direction = direction;
2239         dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2240         dwc->eps[epnum] = dep;
2241
2242         snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2243                         direction ? "in" : "out");
2244
2245         dep->endpoint.name = dep->name;
2246
2247         if (!(dep->number > 1)) {
2248                 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2249                 dep->endpoint.comp_desc = NULL;
2250         }
2251
2252         spin_lock_init(&dep->lock);
2253
2254         if (num == 0)
2255                 ret = dwc3_gadget_init_control_endpoint(dep);
2256         else if (direction)
2257                 ret = dwc3_gadget_init_in_endpoint(dep);
2258         else
2259                 ret = dwc3_gadget_init_out_endpoint(dep);
2260
2261         if (ret)
2262                 return ret;
2263
2264         dep->endpoint.caps.dir_in = direction;
2265         dep->endpoint.caps.dir_out = !direction;
2266
2267         INIT_LIST_HEAD(&dep->pending_list);
2268         INIT_LIST_HEAD(&dep->started_list);
2269
2270         return 0;
2271 }
2272
2273 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2274 {
2275         u8                              epnum;
2276
2277         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2278
2279         for (epnum = 0; epnum < total; epnum++) {
2280                 int                     ret;
2281
2282                 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2283                 if (ret)
2284                         return ret;
2285         }
2286
2287         return 0;
2288 }
2289
2290 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2291 {
2292         struct dwc3_ep                  *dep;
2293         u8                              epnum;
2294
2295         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2296                 dep = dwc->eps[epnum];
2297                 if (!dep)
2298                         continue;
2299                 /*
2300                  * Physical endpoints 0 and 1 are special; they form the
2301                  * bi-directional USB endpoint 0.
2302                  *
2303                  * For those two physical endpoints, we don't allocate a TRB
2304                  * pool nor do we add them the endpoints list. Due to that, we
2305                  * shouldn't do these two operations otherwise we would end up
2306                  * with all sorts of bugs when removing dwc3.ko.
2307                  */
2308                 if (epnum != 0 && epnum != 1) {
2309                         dwc3_free_trb_pool(dep);
2310                         list_del(&dep->endpoint.ep_list);
2311                 }
2312
2313                 kfree(dep);
2314         }
2315 }
2316
2317 /* -------------------------------------------------------------------------- */
2318
2319 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2320                 struct dwc3_request *req, struct dwc3_trb *trb,
2321                 const struct dwc3_event_depevt *event, int status, int chain)
2322 {
2323         unsigned int            count;
2324
2325         dwc3_ep_inc_deq(dep);
2326
2327         trace_dwc3_complete_trb(dep, trb);
2328
2329         /*
2330          * If we're in the middle of series of chained TRBs and we
2331          * receive a short transfer along the way, DWC3 will skip
2332          * through all TRBs including the last TRB in the chain (the
2333          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2334          * bit and SW has to do it manually.
2335          *
2336          * We're going to do that here to avoid problems of HW trying
2337          * to use bogus TRBs for transfers.
2338          */
2339         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2340                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2341
2342         /*
2343          * If we're dealing with unaligned size OUT transfer, we will be left
2344          * with one TRB pending in the ring. We need to manually clear HWO bit
2345          * from that TRB.
2346          */
2347         if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
2348                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2349                 return 1;
2350         }
2351
2352         count = trb->size & DWC3_TRB_SIZE_MASK;
2353         req->remaining += count;
2354
2355         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2356                 return 1;
2357
2358         if (event->status & DEPEVT_STATUS_SHORT && !chain)
2359                 return 1;
2360
2361         if ((event->status & DEPEVT_STATUS_IOC) &&
2362             (trb->ctrl & DWC3_TRB_CTRL_IOC))
2363                 return 1;
2364
2365         if ((event->status & DEPEVT_STATUS_LST) &&
2366             (trb->ctrl & DWC3_TRB_CTRL_LST))
2367                 return 1;
2368
2369         return 0;
2370 }
2371
2372 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2373                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2374                 int status)
2375 {
2376         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2377         struct scatterlist *sg = req->sg;
2378         struct scatterlist *s;
2379         unsigned int pending = req->num_pending_sgs;
2380         unsigned int i;
2381         int ret = 0;
2382
2383         for_each_sg(sg, s, pending, i) {
2384                 trb = &dep->trb_pool[dep->trb_dequeue];
2385
2386                 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2387                         break;
2388
2389                 req->sg = sg_next(s);
2390                 req->num_pending_sgs--;
2391
2392                 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2393                                 trb, event, status, true);
2394                 if (ret)
2395                         break;
2396         }
2397
2398         return ret;
2399 }
2400
2401 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2402                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2403                 int status)
2404 {
2405         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2406
2407         return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2408                         event, status, false);
2409 }
2410
2411 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2412 {
2413         return req->request.actual == req->request.length;
2414 }
2415
2416 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2417                 const struct dwc3_event_depevt *event,
2418                 struct dwc3_request *req, int status)
2419 {
2420         int ret;
2421
2422         if (req->num_pending_sgs)
2423                 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2424                                 status);
2425         else
2426                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2427                                 status);
2428
2429         if (req->unaligned || req->zero) {
2430                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2431                                 status);
2432                 req->unaligned = false;
2433                 req->zero = false;
2434         }
2435
2436         req->request.actual = req->request.length - req->remaining;
2437
2438         if ((!dwc3_gadget_ep_request_completed(req) &&
2439              req->num_pending_sgs) || req->num_pending_sgs) {
2440                 if (!(event->status &
2441                         (DEPEVT_STATUS_SHORT | DEPEVT_STATUS_LST))) {
2442                         __dwc3_gadget_kick_transfer(dep);
2443                         goto out;
2444                 }
2445         }
2446
2447         dwc3_gadget_giveback(dep, req, status);
2448
2449 out:
2450         return ret;
2451 }
2452
2453 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2454                 const struct dwc3_event_depevt *event, int status)
2455 {
2456         struct dwc3_request     *req;
2457         struct dwc3_request     *tmp;
2458
2459         list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2460                 int ret;
2461
2462                 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2463                                 req, status);
2464                 if (ret)
2465                         break;
2466         }
2467 }
2468
2469 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2470                 const struct dwc3_event_depevt *event)
2471 {
2472         dep->frame_number = event->parameters;
2473 }
2474
2475 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2476                 const struct dwc3_event_depevt *event)
2477 {
2478         struct dwc3             *dwc = dep->dwc;
2479         unsigned                status = 0;
2480         bool                    stop = false;
2481
2482         dwc3_gadget_endpoint_frame_from_event(dep, event);
2483
2484         if (event->status & DEPEVT_STATUS_BUSERR)
2485                 status = -ECONNRESET;
2486
2487         if ((event->status & DEPEVT_STATUS_MISSED_ISOC) &&
2488             usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
2489                 status = -EXDEV;
2490
2491                 if (list_empty(&dep->started_list))
2492                         stop = true;
2493         }
2494
2495         dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2496
2497         if (dep->stream_capable && !list_empty(&dep->started_list))
2498                 __dwc3_gadget_kick_transfer(dep);
2499
2500         if (stop) {
2501                 dwc3_stop_active_transfer(dep, true);
2502                 dep->flags = DWC3_EP_ENABLED;
2503         }
2504
2505         /*
2506          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2507          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2508          */
2509         if (dwc->revision < DWC3_REVISION_183A) {
2510                 u32             reg;
2511                 int             i;
2512
2513                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2514                         dep = dwc->eps[i];
2515
2516                         if (!(dep->flags & DWC3_EP_ENABLED))
2517                                 continue;
2518
2519                         if (!list_empty(&dep->started_list))
2520                                 return;
2521                 }
2522
2523                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2524                 reg |= dwc->u1u2;
2525                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2526
2527                 dwc->u1u2 = 0;
2528         }
2529 }
2530
2531 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2532                 const struct dwc3_event_depevt *event)
2533 {
2534         dwc3_gadget_endpoint_frame_from_event(dep, event);
2535         __dwc3_gadget_start_isoc(dep);
2536 }
2537
2538 static void dwc3_endpoint_stream_event(struct dwc3 *dwc,
2539                                        const struct dwc3_event_depevt *event)
2540 {
2541         struct dwc3_ep          *dep;
2542         struct dwc3_request     *req;
2543         u8                      epnum = event->endpoint_number;
2544         u8                      stream_id;
2545
2546         dep = dwc->eps[epnum];
2547
2548         stream_id = event->parameters;
2549
2550         /* Check for request matching the streamid and delete the timer */
2551         list_for_each_entry(req, &dep->started_list, list) {
2552                 if (req->request.stream_id == stream_id) {
2553                         if (timer_pending(&req->stream_timeout_timer))
2554                                 del_timer(&req->stream_timeout_timer);
2555                         break;
2556                 }
2557         }
2558 }
2559
2560 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2561                 const struct dwc3_event_depevt *event)
2562 {
2563         struct dwc3_ep          *dep;
2564         u8                      epnum = event->endpoint_number;
2565         u8                      cmd;
2566
2567         dep = dwc->eps[epnum];
2568
2569         if (!(dep->flags & DWC3_EP_ENABLED)) {
2570                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2571                         return;
2572
2573                 /* Handle only EPCMDCMPLT when EP disabled */
2574                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2575                         return;
2576         }
2577
2578         if (epnum == 0 || epnum == 1) {
2579                 dwc3_ep0_interrupt(dwc, event);
2580                 return;
2581         }
2582
2583         switch (event->endpoint_event) {
2584         case DWC3_DEPEVT_XFERCOMPLETE:
2585                 if (!dep->stream_capable)
2586                         break;
2587                 dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
2588                 /* Fall Through */
2589         case DWC3_DEPEVT_XFERINPROGRESS:
2590                 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2591                 break;
2592         case DWC3_DEPEVT_XFERNOTREADY:
2593                 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2594                 break;
2595         case DWC3_DEPEVT_STREAMEVT:
2596                 if (event->status == DEPEVT_STREAMEVT_FOUND)
2597                         dwc3_endpoint_stream_event(dwc, event);
2598                 break;
2599         case DWC3_DEPEVT_EPCMDCMPLT:
2600                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2601
2602                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2603                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2604                         wake_up(&dep->wait_end_transfer);
2605                 }
2606                 break;
2607         case DWC3_DEPEVT_RXTXFIFOEVT:
2608                 break;
2609         }
2610 }
2611
2612 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2613 {
2614         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2615                 spin_unlock(&dwc->lock);
2616                 dwc->gadget_driver->disconnect(&dwc->gadget);
2617                 spin_lock(&dwc->lock);
2618         }
2619 }
2620
2621 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2622 {
2623         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2624                 spin_unlock(&dwc->lock);
2625                 dwc->gadget_driver->suspend(&dwc->gadget);
2626                 spin_lock(&dwc->lock);
2627         }
2628 }
2629
2630 static void dwc3_resume_gadget(struct dwc3 *dwc)
2631 {
2632         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2633                 spin_unlock(&dwc->lock);
2634                 dwc->gadget_driver->resume(&dwc->gadget);
2635                 spin_lock(&dwc->lock);
2636         }
2637 }
2638
2639 static void dwc3_reset_gadget(struct dwc3 *dwc)
2640 {
2641         if (!dwc->gadget_driver)
2642                 return;
2643
2644         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2645                 spin_unlock(&dwc->lock);
2646                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2647                 spin_lock(&dwc->lock);
2648         }
2649 }
2650
2651 void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force)
2652 {
2653         struct dwc3 *dwc = dep->dwc;
2654         struct dwc3_gadget_ep_cmd_params params;
2655         u32 cmd;
2656         int ret;
2657
2658         if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2659             !dep->resource_index)
2660                 return;
2661
2662         /*
2663          * NOTICE: We are violating what the Databook says about the
2664          * EndTransfer command. Ideally we would _always_ wait for the
2665          * EndTransfer Command Completion IRQ, but that's causing too
2666          * much trouble synchronizing between us and gadget driver.
2667          *
2668          * We have discussed this with the IP Provider and it was
2669          * suggested to giveback all requests here, but give HW some
2670          * extra time to synchronize with the interconnect. We're using
2671          * an arbitrary 100us delay for that.
2672          *
2673          * Note also that a similar handling was tested by Synopsys
2674          * (thanks a lot Paul) and nothing bad has come out of it.
2675          * In short, what we're doing is:
2676          *
2677          * - Issue EndTransfer WITH CMDIOC bit set
2678          * - Wait 100us
2679          *
2680          * As of IP version 3.10a of the DWC_usb3 IP, the controller
2681          * supports a mode to work around the above limitation. The
2682          * software can poll the CMDACT bit in the DEPCMD register
2683          * after issuing a EndTransfer command. This mode is enabled
2684          * by writing GUCTL2[14]. This polling is already done in the
2685          * dwc3_send_gadget_ep_cmd() function so if the mode is
2686          * enabled, the EndTransfer command will have completed upon
2687          * returning from this function and we don't need to delay for
2688          * 100us.
2689          *
2690          * This mode is NOT available on the DWC_usb31 IP.
2691          */
2692
2693         cmd = DWC3_DEPCMD_ENDTRANSFER;
2694         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2695         cmd |= DWC3_DEPCMD_CMDIOC;
2696         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2697         memset(&params, 0, sizeof(params));
2698         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2699         WARN_ON_ONCE(ret);
2700         dep->resource_index = 0;
2701
2702         /*
2703          * when transfer is stopped with force rm bit false, it can be
2704          * restarted by passing resource_index in params; don't loose it
2705          */
2706         if (force)
2707                 dep->resource_index = 0;
2708
2709         if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2710                 /*
2711                  * CMD COMPLETE interrupt is not getting generated for isoc
2712                  * endpoints, so don't set DWC3_EP_END_TRANSFER_PENDING flag
2713                  */
2714                 if (!usb_endpoint_xfer_isoc(dep->endpoint.desc))
2715                         dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2716
2717                 udelay(100);
2718         }
2719 }
2720
2721 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2722 {
2723         u32 epnum;
2724
2725         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2726                 struct dwc3_ep *dep;
2727                 int ret;
2728
2729                 dep = dwc->eps[epnum];
2730                 if (!dep)
2731                         continue;
2732
2733                 if (!(dep->flags & DWC3_EP_STALL))
2734                         continue;
2735
2736                 dep->flags &= ~DWC3_EP_STALL;
2737
2738                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2739                 WARN_ON_ONCE(ret);
2740         }
2741 }
2742
2743 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2744 {
2745         int                     reg;
2746
2747         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2748         reg &= ~DWC3_DCTL_INITU1ENA;
2749         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2750
2751         reg &= ~DWC3_DCTL_INITU2ENA;
2752         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2753
2754         dwc3_disconnect_gadget(dwc);
2755
2756         /* In USB 2.0, to avoid hibernation interrupt at the time of connection
2757          * clear DWC3_DCTL_KEEP_CONNECT bit.
2758          */
2759         if (dwc->has_hibernation) {
2760                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2761                 reg &= ~DWC3_DCTL_KEEP_CONNECT;
2762                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2763         }
2764
2765         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2766         dwc->setup_packet_pending = false;
2767         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2768
2769         dwc->connected = false;
2770 }
2771
2772 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2773 {
2774         u32                     reg;
2775
2776         dwc->connected = true;
2777
2778         /*
2779          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2780          * would cause a missing Disconnect Event if there's a
2781          * pending Setup Packet in the FIFO.
2782          *
2783          * There's no suggested workaround on the official Bug
2784          * report, which states that "unless the driver/application
2785          * is doing any special handling of a disconnect event,
2786          * there is no functional issue".
2787          *
2788          * Unfortunately, it turns out that we _do_ some special
2789          * handling of a disconnect event, namely complete all
2790          * pending transfers, notify gadget driver of the
2791          * disconnection, and so on.
2792          *
2793          * Our suggested workaround is to follow the Disconnect
2794          * Event steps here, instead, based on a setup_packet_pending
2795          * flag. Such flag gets set whenever we have a SETUP_PENDING
2796          * status for EP0 TRBs and gets cleared on XferComplete for the
2797          * same endpoint.
2798          *
2799          * Refers to:
2800          *
2801          * STAR#9000466709: RTL: Device : Disconnect event not
2802          * generated if setup packet pending in FIFO
2803          */
2804         if (dwc->revision < DWC3_REVISION_188A) {
2805                 if (dwc->setup_packet_pending)
2806                         dwc3_gadget_disconnect_interrupt(dwc);
2807         }
2808
2809         dwc3_reset_gadget(dwc);
2810
2811         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2812         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2813         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2814         dwc->test_mode = false;
2815         dwc3_clear_stall_all_ep(dwc);
2816
2817         /* Reset device address to zero */
2818         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2819         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2820         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2821 }
2822
2823 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2824 {
2825         struct dwc3_ep          *dep;
2826         int                     ret;
2827         u32                     reg;
2828         u8                      speed;
2829
2830         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2831         speed = reg & DWC3_DSTS_CONNECTSPD;
2832         dwc->speed = speed;
2833
2834         /*
2835          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2836          * each time on Connect Done.
2837          *
2838          * Currently we always use the reset value. If any platform
2839          * wants to set this to a different value, we need to add a
2840          * setting and update GCTL.RAMCLKSEL here.
2841          */
2842
2843         switch (speed) {
2844         case DWC3_DSTS_SUPERSPEED_PLUS:
2845                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2846                 dwc->gadget.ep0->maxpacket = 512;
2847                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2848                 break;
2849         case DWC3_DSTS_SUPERSPEED:
2850                 /*
2851                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2852                  * would cause a missing USB3 Reset event.
2853                  *
2854                  * In such situations, we should force a USB3 Reset
2855                  * event by calling our dwc3_gadget_reset_interrupt()
2856                  * routine.
2857                  *
2858                  * Refers to:
2859                  *
2860                  * STAR#9000483510: RTL: SS : USB3 reset event may
2861                  * not be generated always when the link enters poll
2862                  */
2863                 if (dwc->revision < DWC3_REVISION_190A)
2864                         dwc3_gadget_reset_interrupt(dwc);
2865
2866                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2867                 dwc->gadget.ep0->maxpacket = 512;
2868                 dwc->gadget.speed = USB_SPEED_SUPER;
2869                 break;
2870         case DWC3_DSTS_HIGHSPEED:
2871                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2872                 dwc->gadget.ep0->maxpacket = 64;
2873                 dwc->gadget.speed = USB_SPEED_HIGH;
2874                 break;
2875         case DWC3_DSTS_FULLSPEED:
2876                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2877                 dwc->gadget.ep0->maxpacket = 64;
2878                 dwc->gadget.speed = USB_SPEED_FULL;
2879                 break;
2880         case DWC3_DSTS_LOWSPEED:
2881                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2882                 dwc->gadget.ep0->maxpacket = 8;
2883                 dwc->gadget.speed = USB_SPEED_LOW;
2884                 break;
2885         }
2886
2887         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2888
2889         /* Enable USB2 LPM Capability */
2890
2891         if ((dwc->revision > DWC3_REVISION_194A) &&
2892             (speed != DWC3_DSTS_SUPERSPEED) &&
2893             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2894                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2895                 reg |= DWC3_DCFG_LPM_CAP;
2896                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2897
2898                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2899                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2900
2901                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2902
2903                 /*
2904                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2905                  * DCFG.LPMCap is set, core responses with an ACK and the
2906                  * BESL value in the LPM token is less than or equal to LPM
2907                  * NYET threshold.
2908                  */
2909                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2910                                 && dwc->has_lpm_erratum,
2911                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2912
2913                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2914                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2915
2916                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2917         } else {
2918                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2919                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2920                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2921         }
2922
2923         dep = dwc->eps[0];
2924         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2925         if (ret) {
2926                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2927                 return;
2928         }
2929
2930         dep = dwc->eps[1];
2931         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2932         if (ret) {
2933                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2934                 return;
2935         }
2936
2937         /*
2938          * In USB 2.0, to avoid hibernation interrupt at the time of connection
2939          * set DWC3_DCTL_KEEP_CONNECT bit here
2940          */
2941         if (dwc->has_hibernation) {
2942                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2943                 reg |= DWC3_DCTL_KEEP_CONNECT;
2944                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2945         }
2946
2947         /*
2948          * Configure PHY via GUSB3PIPECTLn if required.
2949          *
2950          * Update GTXFIFOSIZn
2951          *
2952          * In both cases reset values should be sufficient.
2953          */
2954 }
2955
2956 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2957 {
2958         /*
2959          * TODO take core out of low power mode when that's
2960          * implemented.
2961          */
2962
2963         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2964                 spin_unlock(&dwc->lock);
2965                 dwc->gadget_driver->resume(&dwc->gadget);
2966                 spin_lock(&dwc->lock);
2967         }
2968 }
2969
2970 static irqreturn_t wakeup_interrupt(int irq, void *_dwc)
2971 {
2972         struct dwc3 *dwc = (struct dwc3 *)_dwc;
2973
2974         spin_lock(&dwc->lock);
2975         gadget_wakeup_interrupt(dwc);
2976         spin_unlock(&dwc->lock);
2977
2978         return IRQ_HANDLED;
2979 }
2980
2981 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2982                 unsigned int evtinfo)
2983 {
2984         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2985         unsigned int            pwropt;
2986
2987         /*
2988          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2989          * Hibernation mode enabled which would show up when device detects
2990          * host-initiated U3 exit.
2991          *
2992          * In that case, device will generate a Link State Change Interrupt
2993          * from U3 to RESUME which is only necessary if Hibernation is
2994          * configured in.
2995          *
2996          * There are no functional changes due to such spurious event and we
2997          * just need to ignore it.
2998          *
2999          * Refers to:
3000          *
3001          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
3002          * operational mode
3003          */
3004         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
3005         if ((dwc->revision < DWC3_REVISION_250A) &&
3006                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
3007                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
3008                                 (next == DWC3_LINK_STATE_RESUME)) {
3009                         return;
3010                 }
3011         }
3012
3013         /*
3014          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
3015          * on the link partner, the USB session might do multiple entry/exit
3016          * of low power states before a transfer takes place.
3017          *
3018          * Due to this problem, we might experience lower throughput. The
3019          * suggested workaround is to disable DCTL[12:9] bits if we're
3020          * transitioning from U1/U2 to U0 and enable those bits again
3021          * after a transfer completes and there are no pending transfers
3022          * on any of the enabled endpoints.
3023          *
3024          * This is the first half of that workaround.
3025          *
3026          * Refers to:
3027          *
3028          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
3029          * core send LGO_Ux entering U0
3030          */
3031         if (dwc->revision < DWC3_REVISION_183A) {
3032                 if (next == DWC3_LINK_STATE_U0) {
3033                         u32     u1u2;
3034                         u32     reg;
3035
3036                         switch (dwc->link_state) {
3037                         case DWC3_LINK_STATE_U1:
3038                         case DWC3_LINK_STATE_U2:
3039                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3040                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
3041                                                 | DWC3_DCTL_ACCEPTU2ENA
3042                                                 | DWC3_DCTL_INITU1ENA
3043                                                 | DWC3_DCTL_ACCEPTU1ENA);
3044
3045                                 if (!dwc->u1u2)
3046                                         dwc->u1u2 = reg & u1u2;
3047
3048                                 reg &= ~u1u2;
3049
3050                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3051                                 break;
3052                         default:
3053                                 /* do nothing */
3054                                 break;
3055                         }
3056                 }
3057         }
3058
3059         switch (next) {
3060         case DWC3_LINK_STATE_U1:
3061                 if (dwc->speed == USB_SPEED_SUPER)
3062                         dwc3_suspend_gadget(dwc);
3063                 break;
3064         case DWC3_LINK_STATE_U2:
3065         case DWC3_LINK_STATE_U3:
3066                 dwc3_suspend_gadget(dwc);
3067                 break;
3068         case DWC3_LINK_STATE_RESUME:
3069                 dwc3_resume_gadget(dwc);
3070                 break;
3071         default:
3072                 /* do nothing */
3073                 break;
3074         }
3075
3076         dwc->link_state = next;
3077 }
3078
3079 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
3080                                           unsigned int evtinfo)
3081 {
3082         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
3083
3084         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
3085                 dwc3_suspend_gadget(dwc);
3086
3087         dwc->link_state = next;
3088 }
3089
3090 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
3091                 unsigned int evtinfo)
3092 {
3093         unsigned int is_ss = evtinfo & BIT(4);
3094
3095         /*
3096          * WORKAROUND: DWC3 revison 2.20a with hibernation support
3097          * have a known issue which can cause USB CV TD.9.23 to fail
3098          * randomly.
3099          *
3100          * Because of this issue, core could generate bogus hibernation
3101          * events which SW needs to ignore.
3102          *
3103          * Refers to:
3104          *
3105          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
3106          * Device Fallback from SuperSpeed
3107          */
3108         if ((!!is_ss ^ (dwc->speed >= DWC3_DSTS_SUPERSPEED)) &&
3109             (!(dwc->has_hibernation)))
3110                 return;
3111
3112         /* enter hibernation here */
3113         gadget_hibernation_interrupt(dwc);
3114 }
3115
3116 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
3117                 const struct dwc3_event_devt *event)
3118 {
3119         switch (event->type) {
3120         case DWC3_DEVICE_EVENT_DISCONNECT:
3121                 dwc3_gadget_disconnect_interrupt(dwc);
3122                 break;
3123         case DWC3_DEVICE_EVENT_RESET:
3124                 dwc3_gadget_reset_interrupt(dwc);
3125                 break;
3126         case DWC3_DEVICE_EVENT_CONNECT_DONE:
3127                 dwc3_gadget_conndone_interrupt(dwc);
3128                 break;
3129         case DWC3_DEVICE_EVENT_WAKEUP:
3130                 dwc3_gadget_wakeup_interrupt(dwc);
3131                 break;
3132         case DWC3_DEVICE_EVENT_HIBER_REQ:
3133                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
3134                                         "unexpected hibernation event\n"))
3135                         break;
3136
3137                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
3138                 break;
3139         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
3140                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
3141                 break;
3142         case DWC3_DEVICE_EVENT_EOPF:
3143                 /* It changed to be suspend event for version 2.30a and above */
3144                 if (dwc->revision >= DWC3_REVISION_230A) {
3145                         /*
3146                          * Ignore suspend event until the gadget enters into
3147                          * USB_STATE_CONFIGURED state.
3148                          */
3149                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
3150                                 dwc3_gadget_suspend_interrupt(dwc,
3151                                                 event->event_info);
3152                 }
3153                 break;
3154         case DWC3_DEVICE_EVENT_SOF:
3155         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
3156         case DWC3_DEVICE_EVENT_CMD_CMPL:
3157         case DWC3_DEVICE_EVENT_OVERFLOW:
3158                 break;
3159         default:
3160                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
3161         }
3162 }
3163
3164 static void dwc3_process_event_entry(struct dwc3 *dwc,
3165                 const union dwc3_event *event)
3166 {
3167         trace_dwc3_event(event->raw, dwc);
3168
3169         if (!event->type.is_devspec)
3170                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3171         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3172                 dwc3_gadget_interrupt(dwc, &event->devt);
3173         else
3174                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3175 }
3176
3177 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3178 {
3179         struct dwc3 *dwc = evt->dwc;
3180         irqreturn_t ret = IRQ_NONE;
3181         int left;
3182         u32 reg;
3183
3184         left = evt->count;
3185
3186         if (!(evt->flags & DWC3_EVENT_PENDING))
3187                 return IRQ_NONE;
3188
3189         while (left > 0) {
3190                 union dwc3_event event;
3191
3192                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3193
3194                 dwc3_process_event_entry(dwc, &event);
3195
3196                 /*
3197                  * FIXME we wrap around correctly to the next entry as
3198                  * almost all entries are 4 bytes in size. There is one
3199                  * entry which has 12 bytes which is a regular entry
3200                  * followed by 8 bytes data. ATM I don't know how
3201                  * things are organized if we get next to the a
3202                  * boundary so I worry about that once we try to handle
3203                  * that.
3204                  */
3205                 evt->lpos = (evt->lpos + 4) % evt->length;
3206                 left -= 4;
3207
3208                 if (dwc->is_hibernated)
3209                         break;
3210         }
3211
3212         evt->count = 0;
3213         evt->flags &= ~DWC3_EVENT_PENDING;
3214         ret = IRQ_HANDLED;
3215
3216         if (dwc->is_hibernated)
3217                 return ret;
3218
3219         /* Unmask interrupt */
3220         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3221         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3222         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3223
3224         if (dwc->imod_interval) {
3225                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3226                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3227         }
3228
3229         return ret;
3230 }
3231
3232 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3233 {
3234         struct dwc3_event_buffer *evt = _evt;
3235         struct dwc3 *dwc = evt->dwc;
3236         unsigned long flags;
3237         irqreturn_t ret = IRQ_NONE;
3238
3239         spin_lock_irqsave(&dwc->lock, flags);
3240         ret = dwc3_process_event_buf(evt);
3241         spin_unlock_irqrestore(&dwc->lock, flags);
3242
3243         return ret;
3244 }
3245
3246 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3247 {
3248         struct dwc3 *dwc = evt->dwc;
3249         u32 amount;
3250         u32 count;
3251         u32 reg;
3252
3253         if (pm_runtime_suspended(dwc->dev)) {
3254                 pm_runtime_get(dwc->dev);
3255                 disable_irq_nosync(dwc->irq_gadget);
3256                 dwc->pending_events = true;
3257                 return IRQ_HANDLED;
3258         }
3259
3260         if (dwc->is_hibernated)
3261                 return IRQ_HANDLED;
3262
3263         /*
3264          * With PCIe legacy interrupt, test shows that top-half irq handler can
3265          * be called again after HW interrupt deassertion. Check if bottom-half
3266          * irq event handler completes before caching new event to prevent
3267          * losing events.
3268          */
3269         if (evt->flags & DWC3_EVENT_PENDING)
3270                 return IRQ_HANDLED;
3271
3272         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3273         count &= DWC3_GEVNTCOUNT_MASK;
3274         if (!count)
3275                 return IRQ_NONE;
3276
3277         evt->count = count;
3278         evt->flags |= DWC3_EVENT_PENDING;
3279
3280         /* Mask interrupt */
3281         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3282         reg |= DWC3_GEVNTSIZ_INTMASK;
3283         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3284
3285         amount = min(count, evt->length - evt->lpos);
3286         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3287
3288         if (amount < count)
3289                 memcpy(evt->cache, evt->buf, count - amount);
3290
3291         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3292
3293         return IRQ_WAKE_THREAD;
3294 }
3295
3296 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3297 {
3298         struct dwc3_event_buffer        *evt = _evt;
3299
3300         return dwc3_check_event_buf(evt);
3301 }
3302
3303 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3304 {
3305         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3306         int irq;
3307
3308         irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3309         if (irq > 0)
3310                 goto out;
3311
3312         if (irq == -EPROBE_DEFER)
3313                 goto out;
3314
3315         irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3316         if (irq > 0)
3317                 goto out;
3318
3319         if (irq == -EPROBE_DEFER)
3320                 goto out;
3321
3322         irq = platform_get_irq(dwc3_pdev, 0);
3323         if (irq > 0)
3324                 dwc->irq_gadget = irq;
3325
3326         if (irq == -EPROBE_DEFER)
3327                 goto out;
3328
3329         /* look for wakeup interrupt if hibernation is supported */
3330         if (dwc->has_hibernation) {
3331                 irq = platform_get_irq(dwc3_pdev, 2);
3332                 if (irq > 0)
3333                         dwc->irq_wakeup = irq;
3334
3335                 if (irq == -EPROBE_DEFER)
3336                         goto out;
3337         }
3338
3339         if (irq <= 0) {
3340                 if (irq != -EPROBE_DEFER)
3341                         dev_err(dwc->dev, "missing peripheral IRQ\n");
3342
3343                 if (!irq)
3344                         irq = -EINVAL;
3345         }
3346 out:
3347         return irq;
3348 }
3349
3350 /**
3351  * dwc3_gadget_init - initializes gadget related registers
3352  * @dwc: pointer to our controller context structure
3353  *
3354  * Returns 0 on success otherwise negative errno.
3355  */
3356 int dwc3_gadget_init(struct dwc3 *dwc)
3357 {
3358         int ret;
3359         int irq;
3360
3361         irq = dwc3_gadget_get_irq(dwc);
3362         if (irq < 0) {
3363                 ret = irq;
3364                 goto err0;
3365         }
3366
3367         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3368                                           sizeof(*dwc->ep0_trb) * 2,
3369                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3370         if (!dwc->ep0_trb) {
3371                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3372                 ret = -ENOMEM;
3373                 goto err0;
3374         }
3375
3376         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3377         if (!dwc->setup_buf) {
3378                 ret = -ENOMEM;
3379                 goto err1;
3380         }
3381
3382         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3383                         &dwc->bounce_addr, GFP_KERNEL);
3384         if (!dwc->bounce) {
3385                 ret = -ENOMEM;
3386                 goto err2;
3387         }
3388
3389         init_completion(&dwc->ep0_in_setup);
3390
3391         dwc->gadget.ops                 = &dwc3_gadget_ops;
3392         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3393         dwc->gadget.sg_supported        = true;
3394         dwc->gadget.name                = "dwc3-gadget";
3395         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
3396
3397         /*
3398          * FIXME We might be setting max_speed to <SUPER, however versions
3399          * <2.20a of dwc3 have an issue with metastability (documented
3400          * elsewhere in this driver) which tells us we can't set max speed to
3401          * anything lower than SUPER.
3402          *
3403          * Because gadget.max_speed is only used by composite.c and function
3404          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3405          * to happen so we avoid sending SuperSpeed Capability descriptor
3406          * together with our BOS descriptor as that could confuse host into
3407          * thinking we can handle super speed.
3408          *
3409          * Note that, in fact, we won't even support GetBOS requests when speed
3410          * is less than super speed because we don't have means, yet, to tell
3411          * composite.c that we are USB 2.0 + LPM ECN.
3412          */
3413         if (dwc->revision < DWC3_REVISION_220A &&
3414             !dwc->dis_metastability_quirk)
3415                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3416                                 dwc->revision);
3417
3418         dwc->gadget.max_speed           = dwc->maximum_speed;
3419
3420         /*
3421          * REVISIT: Here we should clear all pending IRQs to be
3422          * sure we're starting from a well known location.
3423          */
3424
3425         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3426         if (ret)
3427                 goto err3;
3428
3429         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3430         if (ret) {
3431                 dev_err(dwc->dev, "failed to register udc\n");
3432                 goto err4;
3433         }
3434
3435         if (dwc->dr_mode == USB_DR_MODE_OTG) {
3436                 struct usb_phy *phy;
3437
3438                 phy = usb_get_phy(USB_PHY_TYPE_USB3);
3439                 if (!IS_ERR(phy)) {
3440                         if (phy && phy->otg) {
3441                                 ret = otg_set_peripheral(phy->otg,
3442                                                          &dwc->gadget);
3443                                 if (ret) {
3444                                         dev_err(dwc->dev,
3445                                                 "otg_set_peripheral failed\n");
3446                                         usb_put_phy(phy);
3447                                         phy = NULL;
3448                                         goto err4;
3449                                 }
3450                         } else {
3451                                 usb_put_phy(phy);
3452                                 phy = NULL;
3453                         }
3454                 }
3455         }
3456
3457         return 0;
3458
3459 err4:
3460         dwc3_gadget_free_endpoints(dwc);
3461
3462 err3:
3463         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3464                         dwc->bounce_addr);
3465
3466 err2:
3467         kfree(dwc->setup_buf);
3468
3469 err1:
3470         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3471                         dwc->ep0_trb, dwc->ep0_trb_addr);
3472
3473 err0:
3474         return ret;
3475 }
3476
3477 /* -------------------------------------------------------------------------- */
3478
3479 void dwc3_gadget_exit(struct dwc3 *dwc)
3480 {
3481         usb_del_gadget_udc(&dwc->gadget);
3482         dwc3_gadget_free_endpoints(dwc);
3483         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3484                           dwc->bounce_addr);
3485         kfree(dwc->setup_buf);
3486         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3487                           dwc->ep0_trb, dwc->ep0_trb_addr);
3488 }
3489
3490 int dwc3_gadget_suspend(struct dwc3 *dwc)
3491 {
3492         if (!dwc->gadget_driver)
3493                 return 0;
3494
3495         if (dwc->is_hibernated) {
3496                 /*
3497                  * As we are about to suspend, wake the controller from
3498                  * D3 & hibernation states
3499                  */
3500                 dwc->force_hiber_wake = true;
3501                 gadget_wakeup_interrupt(dwc);
3502                 dwc->force_hiber_wake = false;
3503         }
3504
3505         dwc3_gadget_run_stop(dwc, false, false);
3506         dwc3_disconnect_gadget(dwc);
3507         __dwc3_gadget_stop(dwc);
3508
3509         return 0;
3510 }
3511
3512 int dwc3_gadget_resume(struct dwc3 *dwc)
3513 {
3514         int                     ret;
3515         u32                     reg;
3516
3517         if (!dwc->gadget_driver)
3518                 return 0;
3519
3520         ret = __dwc3_gadget_start(dwc);
3521         if (ret < 0)
3522                 goto err0;
3523
3524         ret = dwc3_gadget_run_stop(dwc, true, false);
3525         if (ret < 0)
3526                 goto err1;
3527
3528         /* In USB 2.0, to avoid hibernation interrupt at the time of connection
3529          * set DWC3_DCTL_KEEP_CONNECT bit.
3530          */
3531         if (dwc->has_hibernation) {
3532                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
3533                 reg |= DWC3_DCTL_KEEP_CONNECT;
3534                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
3535         }
3536
3537         return 0;
3538
3539 err1:
3540         __dwc3_gadget_stop(dwc);
3541
3542 err0:
3543         return ret;
3544 }
3545
3546 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3547 {
3548         if (dwc->pending_events) {
3549                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3550                 dwc->pending_events = false;
3551                 enable_irq(dwc->irq_gadget);
3552         }
3553 }