drivers/usb/dwc3/gadget.c

   1 // SPDX-License-Identifier: GPL-2.0
   2 /*
   3  * gadget.c - DesignWare USB3 DRD Controller Gadget Framework Link
   4  *
   5  * Copyright (C) 2010-2011 Texas Instruments Incorporated - http://www.ti.com
   6  *
   7  * Authors: Felipe Balbi <balbi@ti.com>,
   8  *          Sebastian Andrzej Siewior <bigeasy@linutronix.de>
   9  */
  10
  11 #include <linux/kernel.h>
  12 #include <linux/delay.h>
  13 #include <linux/slab.h>
  14 #include <linux/spinlock.h>
  15 #include <linux/platform_device.h>
  16 #include <linux/pm_runtime.h>
  17 #include <linux/interrupt.h>
  18 #include <linux/io.h>
  19 #include <linux/list.h>
  20 #include <linux/dma-mapping.h>
  21
  22 #include <linux/usb/ch9.h>
  23 #include <linux/usb/gadget.h>
  24
  25 #include "debug.h"
  26 #include "core.h"
  27 #include "gadget.h"
  28 #include "io.h"
  29
  30 /**
  31  * dwc3_gadget_set_test_mode - enables usb2 test modes
  32  * @dwc: pointer to our context structure
  33  * @mode: the mode to set (J, K SE0 NAK, Force Enable)
  34  *
  35  * Caller should take care of locking. This function will return 0 on
  36  * success or -EINVAL if wrong Test Selector is passed.
  37  */
  38 int dwc3_gadget_set_test_mode(struct dwc3 *dwc, int mode)
  39 {
  40         u32             reg;
  41
  42         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
  43         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
  44
  45         switch (mode) {
  46         case TEST_J:
  47         case TEST_K:
  48         case TEST_SE0_NAK:
  49         case TEST_PACKET:
  50         case TEST_FORCE_EN:
  51                 reg |= mode << 1;
  52                 break;
  53         default:
  54                 return -EINVAL;
  55         }
  56
  57         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
  58
  59         return 0;
  60 }
  61
  62 /**
  63  * dwc3_gadget_get_link_state - gets current state of usb link
  64  * @dwc: pointer to our context structure
  65  *
  66  * Caller should take care of locking. This function will
  67  * return the link state on success (>= 0) or -ETIMEDOUT.
  68  */
  69 int dwc3_gadget_get_link_state(struct dwc3 *dwc)
  70 {
  71         u32             reg;
  72
  73         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  74
  75         return DWC3_DSTS_USBLNKST(reg);
  76 }
  77
  78 /**
  79  * dwc3_gadget_set_link_state - sets usb link to a particular state
  80  * @dwc: pointer to our context structure
  81  * @state: the state to put link into
  82  *
  83  * Caller should take care of locking. This function will
  84  * return 0 on success or -ETIMEDOUT.
  85  */
  86 int dwc3_gadget_set_link_state(struct dwc3 *dwc, enum dwc3_link_state state)
  87 {
  88         int             retries = 10000;
  89         u32             reg;
  90
  91         /*
  92          * Wait until device controller is ready. Only applies to 1.94a and
  93          * later RTL.
  94          */
  95         if (dwc->revision >= DWC3_REVISION_194A) {
  96                 while (--retries) {
  97                         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
  98                         if (reg & DWC3_DSTS_DCNRD)
  99                                 udelay(5);
 100                         else
 101                                 break;
 102                 }
 103
 104                 if (retries <= 0)
 105                         return -ETIMEDOUT;
 106         }
 107
 108         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
 109         reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
 110
 111         /* set requested state */
 112         reg |= DWC3_DCTL_ULSTCHNGREQ(state);
 113         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
 114
 115         /*
 116          * The following code is racy when called from dwc3_gadget_wakeup,
 117          * and is not needed, at least on newer versions
 118          */
 119         if (dwc->revision >= DWC3_REVISION_194A)
 120                 return 0;
 121
 122         /* wait for a change in DSTS */
 123         retries = 10000;
 124         while (--retries) {
 125                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
 126
 127                 if (DWC3_DSTS_USBLNKST(reg) == state)
 128                         return 0;
 129
 130                 udelay(5);
 131         }
 132
 133         return -ETIMEDOUT;
 134 }
 135
 136 /**
 137  * dwc3_ep_inc_trb - increment a trb index.
 138  * @index: Pointer to the TRB index to increment.
 139  *
 140  * The index should never point to the link TRB. After incrementing,
 141  * if it is point to the link TRB, wrap around to the beginning. The
 142  * link TRB is always at the last TRB entry.
 143  */
 144 static void dwc3_ep_inc_trb(u8 *index)
 145 {
 146         (*index)++;
 147         if (*index == (DWC3_TRB_NUM - 1))
 148                 *index = 0;
 149 }
 150
 151 /**
 152  * dwc3_ep_inc_enq - increment endpoint's enqueue pointer
 153  * @dep: The endpoint whose enqueue pointer we're incrementing
 154  */
 155 static void dwc3_ep_inc_enq(struct dwc3_ep *dep)
 156 {
 157         dwc3_ep_inc_trb(&dep->trb_enqueue);
 158 }
 159
 160 /**
 161  * dwc3_ep_inc_deq - increment endpoint's dequeue pointer
 162  * @dep: The endpoint whose enqueue pointer we're incrementing
 163  */
 164 static void dwc3_ep_inc_deq(struct dwc3_ep *dep)
 165 {
 166         dwc3_ep_inc_trb(&dep->trb_dequeue);
 167 }
 168
 169 void dwc3_gadget_del_and_unmap_request(struct dwc3_ep *dep,
 170                 struct dwc3_request *req, int status)
 171 {
 172         struct dwc3                     *dwc = dep->dwc;
 173
 174         req->started = false;
 175         list_del(&req->list);
 176         req->remaining = 0;
 177
 178         if (req->request.status == -EINPROGRESS)
 179                 req->request.status = status;
 180
 181         if (req->trb)
 182                 usb_gadget_unmap_request_by_dev(dwc->sysdev,
 183                                 &req->request, req->direction);
 184
 185         req->trb = NULL;
 186         trace_dwc3_gadget_giveback(req);
 187
 188         if (dep->number > 1)
 189                 pm_runtime_put(dwc->dev);
 190 }
 191
 192 /**
 193  * dwc3_gadget_giveback - call struct usb_request's ->complete callback
 194  * @dep: The endpoint to whom the request belongs to
 195  * @req: The request we're giving back
 196  * @status: completion code for the request
 197  *
 198  * Must be called with controller's lock held and interrupts disabled. This
 199  * function will unmap @req and call its ->complete() callback to notify upper
 200  * layers that it has completed.
 201  */
 202 void dwc3_gadget_giveback(struct dwc3_ep *dep, struct dwc3_request *req,
 203                 int status)
 204 {
 205         struct dwc3                     *dwc = dep->dwc;
 206
 207         dwc3_gadget_del_and_unmap_request(dep, req, status);
 208
 209         spin_unlock(&dwc->lock);
 210         usb_gadget_giveback_request(&dep->endpoint, &req->request);
 211         spin_lock(&dwc->lock);
 212 }
 213
 214 /**
 215  * dwc3_send_gadget_generic_command - issue a generic command for the controller
 216  * @dwc: pointer to the controller context
 217  * @cmd: the command to be issued
 218  * @param: command parameter
 219  *
 220  * Caller should take care of locking. Issue @cmd with a given @param to @dwc
 221  * and wait for its completion.
 222  */
 223 int dwc3_send_gadget_generic_command(struct dwc3 *dwc, unsigned cmd, u32 param)
 224 {
 225         u32             timeout = 500;
 226         int             status = 0;
 227         int             ret = 0;
 228         u32             reg;
 229
 230         dwc3_writel(dwc->regs, DWC3_DGCMDPAR, param);
 231         dwc3_writel(dwc->regs, DWC3_DGCMD, cmd | DWC3_DGCMD_CMDACT);
 232
 233         do {
 234                 reg = dwc3_readl(dwc->regs, DWC3_DGCMD);
 235                 if (!(reg & DWC3_DGCMD_CMDACT)) {
 236                         status = DWC3_DGCMD_STATUS(reg);
 237                         if (status)
 238                                 ret = -EINVAL;
 239                         break;
 240                 }
 241         } while (--timeout);
 242
 243         if (!timeout) {
 244                 ret = -ETIMEDOUT;
 245                 status = -ETIMEDOUT;
 246         }
 247
 248         trace_dwc3_gadget_generic_cmd(cmd, param, status);
 249
 250         return ret;
 251 }
 252
 253 static int __dwc3_gadget_wakeup(struct dwc3 *dwc);
 254
 255 /**
 256  * dwc3_send_gadget_ep_cmd - issue an endpoint command
 257  * @dep: the endpoint to which the command is going to be issued
 258  * @cmd: the command to be issued
 259  * @params: parameters to the command
 260  *
 261  * Caller should handle locking. This function will issue @cmd with given
 262  * @params to @dep and wait for its completion.
 263  */
 264 int dwc3_send_gadget_ep_cmd(struct dwc3_ep *dep, unsigned cmd,
 265                 struct dwc3_gadget_ep_cmd_params *params)
 266 {
 267         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 268         struct dwc3             *dwc = dep->dwc;
 269         u32                     timeout = 1000;
 270         u32                     reg;
 271
 272         int                     cmd_status = 0;
 273         int                     susphy = false;
 274         int                     ret = -EINVAL;
 275
 276         /*
 277          * Synopsys Databook 2.60a states, on section 6.3.2.5.[1-8], that if
 278          * we're issuing an endpoint command, we must check if
 279          * GUSB2PHYCFG.SUSPHY bit is set. If it is, then we need to clear it.
 280          *
 281          * We will also set SUSPHY bit to what it was before returning as stated
 282          * by the same section on Synopsys databook.
 283          */
 284         if (dwc->gadget.speed <= USB_SPEED_HIGH) {
 285                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 286                 if (unlikely(reg & DWC3_GUSB2PHYCFG_SUSPHY)) {
 287                         susphy = true;
 288                         reg &= ~DWC3_GUSB2PHYCFG_SUSPHY;
 289                         dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 290                 }
 291         }
 292
 293         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_STARTTRANSFER) {
 294                 int             needs_wakeup;
 295
 296                 needs_wakeup = (dwc->link_state == DWC3_LINK_STATE_U1 ||
 297                                 dwc->link_state == DWC3_LINK_STATE_U2 ||
 298                                 dwc->link_state == DWC3_LINK_STATE_U3);
 299
 300                 if (unlikely(needs_wakeup)) {
 301                         ret = __dwc3_gadget_wakeup(dwc);
 302                         dev_WARN_ONCE(dwc->dev, ret, "wakeup failed --> %d\n",
 303                                         ret);
 304                 }
 305         }
 306
 307         dwc3_writel(dep->regs, DWC3_DEPCMDPAR0, params->param0);
 308         dwc3_writel(dep->regs, DWC3_DEPCMDPAR1, params->param1);
 309         dwc3_writel(dep->regs, DWC3_DEPCMDPAR2, params->param2);
 310
 311         /*
 312          * Synopsys Databook 2.60a states in section 6.3.2.5.6 of that if we're
 313          * not relying on XferNotReady, we can make use of a special "No
 314          * Response Update Transfer" command where we should clear both CmdAct
 315          * and CmdIOC bits.
 316          *
 317          * With this, we don't need to wait for command completion and can
 318          * straight away issue further commands to the endpoint.
 319          *
 320          * NOTICE: We're making an assumption that control endpoints will never
 321          * make use of Update Transfer command. This is a safe assumption
 322          * because we can never have more than one request at a time with
 323          * Control Endpoints. If anybody changes that assumption, this chunk
 324          * needs to be updated accordingly.
 325          */
 326         if (DWC3_DEPCMD_CMD(cmd) == DWC3_DEPCMD_UPDATETRANSFER &&
 327                         !usb_endpoint_xfer_isoc(desc))
 328                 cmd &= ~(DWC3_DEPCMD_CMDIOC | DWC3_DEPCMD_CMDACT);
 329         else
 330                 cmd |= DWC3_DEPCMD_CMDACT;
 331
 332         dwc3_writel(dep->regs, DWC3_DEPCMD, cmd);
 333         do {
 334                 reg = dwc3_readl(dep->regs, DWC3_DEPCMD);
 335                 if (!(reg & DWC3_DEPCMD_CMDACT)) {
 336                         cmd_status = DWC3_DEPCMD_STATUS(reg);
 337
 338                         switch (cmd_status) {
 339                         case 0:
 340                                 ret = 0;
 341                                 break;
 342                         case DEPEVT_TRANSFER_NO_RESOURCE:
 343                                 ret = -EINVAL;
 344                                 break;
 345                         case DEPEVT_TRANSFER_BUS_EXPIRY:
 346                                 /*
 347                                  * SW issues START TRANSFER command to
 348                                  * isochronous ep with future frame interval. If
 349                                  * future interval time has already passed when
 350                                  * core receives the command, it will respond
 351                                  * with an error status of 'Bus Expiry'.
 352                                  *
 353                                  * Instead of always returning -EINVAL, let's
 354                                  * give a hint to the gadget driver that this is
 355                                  * the case by returning -EAGAIN.
 356                                  */
 357                                 ret = -EAGAIN;
 358                                 break;
 359                         default:
 360                                 dev_WARN(dwc->dev, "UNKNOWN cmd status\n");
 361                         }
 362
 363                         break;
 364                 }
 365         } while (--timeout);
 366
 367         if (timeout == 0) {
 368                 ret = -ETIMEDOUT;
 369                 cmd_status = -ETIMEDOUT;
 370         }
 371
 372         trace_dwc3_gadget_ep_cmd(dep, cmd, params, cmd_status);
 373
 374         if (ret == 0) {
 375                 switch (DWC3_DEPCMD_CMD(cmd)) {
 376                 case DWC3_DEPCMD_STARTTRANSFER:
 377                         dep->flags |= DWC3_EP_TRANSFER_STARTED;
 378                         break;
 379                 case DWC3_DEPCMD_ENDTRANSFER:
 380                         dep->flags &= ~DWC3_EP_TRANSFER_STARTED;
 381                         break;
 382                 default:
 383                         /* nothing */
 384                         break;
 385                 }
 386         }
 387
 388         if (unlikely(susphy)) {
 389                 reg = dwc3_readl(dwc->regs, DWC3_GUSB2PHYCFG(0));
 390                 reg |= DWC3_GUSB2PHYCFG_SUSPHY;
 391                 dwc3_writel(dwc->regs, DWC3_GUSB2PHYCFG(0), reg);
 392         }
 393
 394         return ret;
 395 }
 396
 397 static int dwc3_send_clear_stall_ep_cmd(struct dwc3_ep *dep)
 398 {
 399         struct dwc3 *dwc = dep->dwc;
 400         struct dwc3_gadget_ep_cmd_params params;
 401         u32 cmd = DWC3_DEPCMD_CLEARSTALL;
 402
 403         /*
 404          * As of core revision 2.60a the recommended programming model
 405          * is to set the ClearPendIN bit when issuing a Clear Stall EP
 406          * command for IN endpoints. This is to prevent an issue where
 407          * some (non-compliant) hosts may not send ACK TPs for pending
 408          * IN transfers due to a mishandled error condition. Synopsys
 409          * STAR 9000614252.
 410          */
 411         if (dep->direction && (dwc->revision >= DWC3_REVISION_260A) &&
 412             (dwc->gadget.speed >= USB_SPEED_SUPER))
 413                 cmd |= DWC3_DEPCMD_CLEARPENDIN;
 414
 415         memset(&params, 0, sizeof(params));
 416
 417         return dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 418 }
 419
 420 static dma_addr_t dwc3_trb_dma_offset(struct dwc3_ep *dep,
 421                 struct dwc3_trb *trb)
 422 {
 423         u32             offset = (char *) trb - (char *) dep->trb_pool;
 424
 425         return dep->trb_pool_dma + offset;
 426 }
 427
 428 static int dwc3_alloc_trb_pool(struct dwc3_ep *dep)
 429 {
 430         struct dwc3             *dwc = dep->dwc;
 431
 432         if (dep->trb_pool)
 433                 return 0;
 434
 435         dep->trb_pool = dma_alloc_coherent(dwc->sysdev,
 436                         sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 437                         &dep->trb_pool_dma, GFP_KERNEL);
 438         if (!dep->trb_pool) {
 439                 dev_err(dep->dwc->dev, "failed to allocate trb pool for %s\n",
 440                                 dep->name);
 441                 return -ENOMEM;
 442         }
 443
 444         return 0;
 445 }
 446
 447 static void dwc3_free_trb_pool(struct dwc3_ep *dep)
 448 {
 449         struct dwc3             *dwc = dep->dwc;
 450
 451         dma_free_coherent(dwc->sysdev, sizeof(struct dwc3_trb) * DWC3_TRB_NUM,
 452                         dep->trb_pool, dep->trb_pool_dma);
 453
 454         dep->trb_pool = NULL;
 455         dep->trb_pool_dma = 0;
 456 }
 457
 458 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep);
 459
 460 /**
 461  * dwc3_gadget_start_config - configure ep resources
 462  * @dwc: pointer to our controller context structure
 463  * @dep: endpoint that is being enabled
 464  *
 465  * Issue a %DWC3_DEPCMD_DEPSTARTCFG command to @dep. After the command's
 466  * completion, it will set Transfer Resource for all available endpoints.
 467  *
 468  * The assignment of transfer resources cannot perfectly follow the data book
 469  * due to the fact that the controller driver does not have all knowledge of the
 470  * configuration in advance. It is given this information piecemeal by the
 471  * composite gadget framework after every SET_CONFIGURATION and
 472  * SET_INTERFACE. Trying to follow the databook programming model in this
 473  * scenario can cause errors. For two reasons:
 474  *
 475  * 1) The databook says to do %DWC3_DEPCMD_DEPSTARTCFG for every
 476  * %USB_REQ_SET_CONFIGURATION and %USB_REQ_SET_INTERFACE (8.1.5). This is
 477  * incorrect in the scenario of multiple interfaces.
 478  *
 479  * 2) The databook does not mention doing more %DWC3_DEPCMD_DEPXFERCFG for new
 480  * endpoint on alt setting (8.1.6).
 481  *
 482  * The following simplified method is used instead:
 483  *
 484  * All hardware endpoints can be assigned a transfer resource and this setting
 485  * will stay persistent until either a core reset or hibernation. So whenever we
 486  * do a %DWC3_DEPCMD_DEPSTARTCFG(0) we can go ahead and do
 487  * %DWC3_DEPCMD_DEPXFERCFG for every hardware endpoint as well. We are
 488  * guaranteed that there are as many transfer resources as endpoints.
 489  *
 490  * This function is called for each endpoint when it is being enabled but is
 491  * triggered only when called for EP0-out, which always happens first, and which
 492  * should only happen in one of the above conditions.
 493  */
 494 static int dwc3_gadget_start_config(struct dwc3_ep *dep)
 495 {
 496         struct dwc3_gadget_ep_cmd_params params;
 497         struct dwc3             *dwc;
 498         u32                     cmd;
 499         int                     i;
 500         int                     ret;
 501
 502         if (dep->number)
 503                 return 0;
 504
 505         memset(&params, 0x00, sizeof(params));
 506         cmd = DWC3_DEPCMD_DEPSTARTCFG;
 507         dwc = dep->dwc;
 508
 509         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 510         if (ret)
 511                 return ret;
 512
 513         for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
 514                 struct dwc3_ep *dep = dwc->eps[i];
 515
 516                 if (!dep)
 517                         continue;
 518
 519                 ret = dwc3_gadget_set_xfer_resource(dep);
 520                 if (ret)
 521                         return ret;
 522         }
 523
 524         return 0;
 525 }
 526
 527 static int dwc3_gadget_set_ep_config(struct dwc3_ep *dep, unsigned int action)
 528 {
 529         const struct usb_ss_ep_comp_descriptor *comp_desc;
 530         const struct usb_endpoint_descriptor *desc;
 531         struct dwc3_gadget_ep_cmd_params params;
 532         struct dwc3 *dwc = dep->dwc;
 533
 534         comp_desc = dep->endpoint.comp_desc;
 535         desc = dep->endpoint.desc;
 536
 537         memset(&params, 0x00, sizeof(params));
 538
 539         params.param0 = DWC3_DEPCFG_EP_TYPE(usb_endpoint_type(desc))
 540                 | DWC3_DEPCFG_MAX_PACKET_SIZE(usb_endpoint_maxp(desc));
 541
 542         /* Burst size is only needed in SuperSpeed mode */
 543         if (dwc->gadget.speed >= USB_SPEED_SUPER) {
 544                 u32 burst = dep->endpoint.maxburst;
 545                 params.param0 |= DWC3_DEPCFG_BURST_SIZE(burst - 1);
 546         }
 547
 548         params.param0 |= action;
 549         if (action == DWC3_DEPCFG_ACTION_RESTORE)
 550                 params.param2 |= dep->saved_state;
 551
 552         if (usb_endpoint_xfer_control(desc))
 553                 params.param1 = DWC3_DEPCFG_XFER_COMPLETE_EN;
 554
 555         if (dep->number <= 1 || usb_endpoint_xfer_isoc(desc))
 556                 params.param1 |= DWC3_DEPCFG_XFER_NOT_READY_EN;
 557
 558         if (usb_ss_max_streams(comp_desc) && usb_endpoint_xfer_bulk(desc)) {
 559                 params.param1 |= DWC3_DEPCFG_STREAM_CAPABLE
 560                         | DWC3_DEPCFG_STREAM_EVENT_EN;
 561                 dep->stream_capable = true;
 562         }
 563
 564         if (!usb_endpoint_xfer_control(desc))
 565                 params.param1 |= DWC3_DEPCFG_XFER_IN_PROGRESS_EN;
 566
 567         /*
 568          * We are doing 1:1 mapping for endpoints, meaning
 569          * Physical Endpoints 2 maps to Logical Endpoint 2 and
 570          * so on. We consider the direction bit as part of the physical
 571          * endpoint number. So USB endpoint 0x81 is 0x03.
 572          */
 573         params.param1 |= DWC3_DEPCFG_EP_NUMBER(dep->number);
 574
 575         /*
 576          * We must use the lower 16 TX FIFOs even though
 577          * HW might have more
 578          */
 579         if (dep->direction)
 580                 params.param0 |= DWC3_DEPCFG_FIFO_NUMBER(dep->number >> 1);
 581
 582         if (desc->bInterval) {
 583                 params.param1 |= DWC3_DEPCFG_BINTERVAL_M1(desc->bInterval - 1);
 584                 dep->interval = 1 << (desc->bInterval - 1);
 585         }
 586
 587         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETEPCONFIG, &params);
 588 }
 589
 590 static int dwc3_gadget_set_xfer_resource(struct dwc3_ep *dep)
 591 {
 592         struct dwc3_gadget_ep_cmd_params params;
 593
 594         memset(&params, 0x00, sizeof(params));
 595
 596         params.param0 = DWC3_DEPXFERCFG_NUM_XFER_RES(1);
 597
 598         return dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETTRANSFRESOURCE,
 599                         &params);
 600 }
 601
 602 /**
 603  * __dwc3_gadget_ep_enable - initializes a hw endpoint
 604  * @dep: endpoint to be initialized
 605  * @action: one of INIT, MODIFY or RESTORE
 606  *
 607  * Caller should take care of locking. Execute all necessary commands to
 608  * initialize a HW endpoint so it can be used by a gadget driver.
 609  */
 610 static int __dwc3_gadget_ep_enable(struct dwc3_ep *dep, unsigned int action)
 611 {
 612         const struct usb_endpoint_descriptor *desc = dep->endpoint.desc;
 613         struct dwc3             *dwc = dep->dwc;
 614
 615         u32                     reg;
 616         int                     ret;
 617
 618         if (!(dep->flags & DWC3_EP_ENABLED)) {
 619                 ret = dwc3_gadget_start_config(dep);
 620                 if (ret)
 621                         return ret;
 622         }
 623
 624         ret = dwc3_gadget_set_ep_config(dep, action);
 625         if (ret)
 626                 return ret;
 627
 628         if (!(dep->flags & DWC3_EP_ENABLED)) {
 629                 struct dwc3_trb *trb_st_hw;
 630                 struct dwc3_trb *trb_link;
 631
 632                 dep->type = usb_endpoint_type(desc);
 633                 dep->flags |= DWC3_EP_ENABLED;
 634                 dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
 635
 636                 reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 637                 reg |= DWC3_DALEPENA_EP(dep->number);
 638                 dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 639
 640                 init_waitqueue_head(&dep->wait_end_transfer);
 641
 642                 if (usb_endpoint_xfer_control(desc))
 643                         goto out;
 644
 645                 /* Initialize the TRB ring */
 646                 dep->trb_dequeue = 0;
 647                 dep->trb_enqueue = 0;
 648                 memset(dep->trb_pool, 0,
 649                        sizeof(struct dwc3_trb) * DWC3_TRB_NUM);
 650
 651                 /* Link TRB. The HWO bit is never reset */
 652                 trb_st_hw = &dep->trb_pool[0];
 653
 654                 trb_link = &dep->trb_pool[DWC3_TRB_NUM - 1];
 655                 trb_link->bpl = lower_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 656                 trb_link->bph = upper_32_bits(dwc3_trb_dma_offset(dep, trb_st_hw));
 657                 trb_link->ctrl |= DWC3_TRBCTL_LINK_TRB;
 658                 trb_link->ctrl |= DWC3_TRB_CTRL_HWO;
 659         }
 660
 661         /*
 662          * Issue StartTransfer here with no-op TRB so we can always rely on No
 663          * Response Update Transfer command.
 664          */
 665         if (usb_endpoint_xfer_bulk(desc) ||
 666                         usb_endpoint_xfer_int(desc)) {
 667                 struct dwc3_gadget_ep_cmd_params params;
 668                 struct dwc3_trb *trb;
 669                 dma_addr_t trb_dma;
 670                 u32 cmd;
 671
 672                 memset(&params, 0, sizeof(params));
 673                 trb = &dep->trb_pool[0];
 674                 trb_dma = dwc3_trb_dma_offset(dep, trb);
 675
 676                 params.param0 = upper_32_bits(trb_dma);
 677                 params.param1 = lower_32_bits(trb_dma);
 678
 679                 cmd = DWC3_DEPCMD_STARTTRANSFER;
 680
 681                 ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
 682                 if (ret < 0)
 683                         return ret;
 684
 685                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
 686                 WARN_ON_ONCE(!dep->resource_index);
 687         }
 688
 689 out:
 690         trace_dwc3_gadget_ep_enable(dep);
 691
 692         return 0;
 693 }
 694
 695 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force);
 696 static void dwc3_remove_requests(struct dwc3 *dwc, struct dwc3_ep *dep)
 697 {
 698         struct dwc3_request             *req;
 699
 700         dwc3_stop_active_transfer(dep, true);
 701
 702         /* - giveback all requests to gadget driver */
 703         while (!list_empty(&dep->started_list)) {
 704                 req = next_request(&dep->started_list);
 705
 706                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 707         }
 708
 709         while (!list_empty(&dep->pending_list)) {
 710                 req = next_request(&dep->pending_list);
 711
 712                 dwc3_gadget_giveback(dep, req, -ESHUTDOWN);
 713         }
 714 }
 715
 716 /**
 717  * __dwc3_gadget_ep_disable - disables a hw endpoint
 718  * @dep: the endpoint to disable
 719  *
 720  * This function undoes what __dwc3_gadget_ep_enable did and also removes
 721  * requests which are currently being processed by the hardware and those which
 722  * are not yet scheduled.
 723  *
 724  * Caller should take care of locking.
 725  */
 726 static int __dwc3_gadget_ep_disable(struct dwc3_ep *dep)
 727 {
 728         struct dwc3             *dwc = dep->dwc;
 729         u32                     reg;
 730
 731         trace_dwc3_gadget_ep_disable(dep);
 732
 733         dwc3_remove_requests(dwc, dep);
 734
 735         /* make sure HW endpoint isn't stalled */
 736         if (dep->flags & DWC3_EP_STALL)
 737                 __dwc3_gadget_ep_set_halt(dep, 0, false);
 738
 739         reg = dwc3_readl(dwc->regs, DWC3_DALEPENA);
 740         reg &= ~DWC3_DALEPENA_EP(dep->number);
 741         dwc3_writel(dwc->regs, DWC3_DALEPENA, reg);
 742
 743         dep->stream_capable = false;
 744         dep->type = 0;
 745         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
 746
 747         /* Clear out the ep descriptors for non-ep0 */
 748         if (dep->number > 1) {
 749                 dep->endpoint.comp_desc = NULL;
 750                 dep->endpoint.desc = NULL;
 751         }
 752
 753         return 0;
 754 }
 755
 756 /* -------------------------------------------------------------------------- */
 757
 758 static int dwc3_gadget_ep0_enable(struct usb_ep *ep,
 759                 const struct usb_endpoint_descriptor *desc)
 760 {
 761         return -EINVAL;
 762 }
 763
 764 static int dwc3_gadget_ep0_disable(struct usb_ep *ep)
 765 {
 766         return -EINVAL;
 767 }
 768
 769 /* -------------------------------------------------------------------------- */
 770
 771 static int dwc3_gadget_ep_enable(struct usb_ep *ep,
 772                 const struct usb_endpoint_descriptor *desc)
 773 {
 774         struct dwc3_ep                  *dep;
 775         struct dwc3                     *dwc;
 776         unsigned long                   flags;
 777         int                             ret;
 778
 779         if (!ep || !desc || desc->bDescriptorType != USB_DT_ENDPOINT) {
 780                 pr_debug("dwc3: invalid parameters\n");
 781                 return -EINVAL;
 782         }
 783
 784         if (!desc->wMaxPacketSize) {
 785                 pr_debug("dwc3: missing wMaxPacketSize\n");
 786                 return -EINVAL;
 787         }
 788
 789         dep = to_dwc3_ep(ep);
 790         dwc = dep->dwc;
 791
 792         if (dev_WARN_ONCE(dwc->dev, dep->flags & DWC3_EP_ENABLED,
 793                                         "%s is already enabled\n",
 794                                         dep->name))
 795                 return 0;
 796
 797         spin_lock_irqsave(&dwc->lock, flags);
 798         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
 799         spin_unlock_irqrestore(&dwc->lock, flags);
 800
 801         return ret;
 802 }
 803
 804 static int dwc3_gadget_ep_disable(struct usb_ep *ep)
 805 {
 806         struct dwc3_ep                  *dep;
 807         struct dwc3                     *dwc;
 808         unsigned long                   flags;
 809         int                             ret;
 810
 811         if (!ep) {
 812                 pr_debug("dwc3: invalid parameters\n");
 813                 return -EINVAL;
 814         }
 815
 816         dep = to_dwc3_ep(ep);
 817         dwc = dep->dwc;
 818
 819         if (dev_WARN_ONCE(dwc->dev, !(dep->flags & DWC3_EP_ENABLED),
 820                                         "%s is already disabled\n",
 821                                         dep->name))
 822                 return 0;
 823
 824         spin_lock_irqsave(&dwc->lock, flags);
 825         ret = __dwc3_gadget_ep_disable(dep);
 826         spin_unlock_irqrestore(&dwc->lock, flags);
 827
 828         return ret;
 829 }
 830
 831 static struct usb_request *dwc3_gadget_ep_alloc_request(struct usb_ep *ep,
 832                 gfp_t gfp_flags)
 833 {
 834         struct dwc3_request             *req;
 835         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
 836
 837         req = kzalloc(sizeof(*req), gfp_flags);
 838         if (!req)
 839                 return NULL;
 840
 841         req->epnum      = dep->number;
 842         req->dep        = dep;
 843
 844         trace_dwc3_alloc_request(req);
 845
 846         return &req->request;
 847 }
 848
 849 static void dwc3_gadget_ep_free_request(struct usb_ep *ep,
 850                 struct usb_request *request)
 851 {
 852         struct dwc3_request             *req = to_dwc3_request(request);
 853
 854         trace_dwc3_free_request(req);
 855         kfree(req);
 856 }
 857
 858 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep);
 859
 860 static void __dwc3_prepare_one_trb(struct dwc3_ep *dep, struct dwc3_trb *trb,
 861                 dma_addr_t dma, unsigned length, unsigned chain, unsigned node,
 862                 unsigned stream_id, unsigned short_not_ok, unsigned no_interrupt)
 863 {
 864         struct dwc3             *dwc = dep->dwc;
 865         struct usb_gadget       *gadget = &dwc->gadget;
 866         enum usb_device_speed   speed = gadget->speed;
 867
 868         dwc3_ep_inc_enq(dep);
 869
 870         trb->size = DWC3_TRB_SIZE_LENGTH(length);
 871         trb->bpl = lower_32_bits(dma);
 872         trb->bph = upper_32_bits(dma);
 873
 874         switch (usb_endpoint_type(dep->endpoint.desc)) {
 875         case USB_ENDPOINT_XFER_CONTROL:
 876                 trb->ctrl = DWC3_TRBCTL_CONTROL_SETUP;
 877                 break;
 878
 879         case USB_ENDPOINT_XFER_ISOC:
 880                 if (!node) {
 881                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS_FIRST;
 882
 883                         /*
 884                          * USB Specification 2.0 Section 5.9.2 states that: "If
 885                          * there is only a single transaction in the microframe,
 886                          * only a DATA0 data packet PID is used.  If there are
 887                          * two transactions per microframe, DATA1 is used for
 888                          * the first transaction data packet and DATA0 is used
 889                          * for the second transaction data packet.  If there are
 890                          * three transactions per microframe, DATA2 is used for
 891                          * the first transaction data packet, DATA1 is used for
 892                          * the second, and DATA0 is used for the third."
 893                          *
 894                          * IOW, we should satisfy the following cases:
 895                          *
 896                          * 1) length <= maxpacket
 897                          *      - DATA0
 898                          *
 899                          * 2) maxpacket < length <= (2 * maxpacket)
 900                          *      - DATA1, DATA0
 901                          *
 902                          * 3) (2 * maxpacket) < length <= (3 * maxpacket)
 903                          *      - DATA2, DATA1, DATA0
 904                          */
 905                         if (speed == USB_SPEED_HIGH) {
 906                                 struct usb_ep *ep = &dep->endpoint;
 907                                 unsigned int mult = 2;
 908                                 unsigned int maxp = usb_endpoint_maxp(ep->desc);
 909
 910                                 if (length <= (2 * maxp))
 911                                         mult--;
 912
 913                                 if (length <= maxp)
 914                                         mult--;
 915
 916                                 trb->size |= DWC3_TRB_SIZE_PCM1(mult);
 917                         }
 918                 } else {
 919                         trb->ctrl = DWC3_TRBCTL_ISOCHRONOUS;
 920                 }
 921
 922                 /* always enable Interrupt on Missed ISOC */
 923                 trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 924                 break;
 925
 926         case USB_ENDPOINT_XFER_BULK:
 927         case USB_ENDPOINT_XFER_INT:
 928                 trb->ctrl = DWC3_TRBCTL_NORMAL;
 929                 break;
 930         default:
 931                 /*
 932                  * This is only possible with faulty memory because we
 933                  * checked it already :)
 934                  */
 935                 dev_WARN(dwc->dev, "Unknown endpoint type %d\n",
 936                                 usb_endpoint_type(dep->endpoint.desc));
 937         }
 938
 939         /* always enable Continue on Short Packet */
 940         if (usb_endpoint_dir_out(dep->endpoint.desc)) {
 941                 trb->ctrl |= DWC3_TRB_CTRL_CSP;
 942
 943                 if (short_not_ok)
 944                         trb->ctrl |= DWC3_TRB_CTRL_ISP_IMI;
 945         }
 946
 947         if ((!no_interrupt && !chain) ||
 948                         (dwc3_calc_trbs_left(dep) == 0))
 949                 trb->ctrl |= DWC3_TRB_CTRL_IOC;
 950
 951         if (chain)
 952                 trb->ctrl |= DWC3_TRB_CTRL_CHN;
 953
 954         if (usb_endpoint_xfer_bulk(dep->endpoint.desc) && dep->stream_capable)
 955                 trb->ctrl |= DWC3_TRB_CTRL_SID_SOFN(stream_id);
 956
 957         trb->ctrl |= DWC3_TRB_CTRL_HWO;
 958
 959         trace_dwc3_prepare_trb(dep, trb);
 960 }
 961
 962 /**
 963  * dwc3_prepare_one_trb - setup one TRB from one request
 964  * @dep: endpoint for which this request is prepared
 965  * @req: dwc3_request pointer
 966  * @chain: should this TRB be chained to the next?
 967  * @node: only for isochronous endpoints. First TRB needs different type.
 968  */
 969 static void dwc3_prepare_one_trb(struct dwc3_ep *dep,
 970                 struct dwc3_request *req, unsigned chain, unsigned node)
 971 {
 972         struct dwc3_trb         *trb;
 973         unsigned int            length;
 974         dma_addr_t              dma;
 975         unsigned                stream_id = req->request.stream_id;
 976         unsigned                short_not_ok = req->request.short_not_ok;
 977         unsigned                no_interrupt = req->request.no_interrupt;
 978
 979         if (req->request.num_sgs > 0) {
 980                 length = sg_dma_len(req->start_sg);
 981                 dma = sg_dma_address(req->start_sg);
 982         } else {
 983                 length = req->request.length;
 984                 dma = req->request.dma;
 985         }
 986
 987         trb = &dep->trb_pool[dep->trb_enqueue];
 988
 989         if (!req->trb) {
 990                 dwc3_gadget_move_started_request(req);
 991                 req->trb = trb;
 992                 req->trb_dma = dwc3_trb_dma_offset(dep, trb);
 993         }
 994
 995         __dwc3_prepare_one_trb(dep, trb, dma, length, chain, node,
 996                         stream_id, short_not_ok, no_interrupt);
 997 }
 998
 999 /**
1000  * dwc3_ep_prev_trb - returns the previous TRB in the ring
1001  * @dep: The endpoint with the TRB ring
1002  * @index: The index of the current TRB in the ring
1003  *
1004  * Returns the TRB prior to the one pointed to by the index. If the
1005  * index is 0, we will wrap backwards, skip the link TRB, and return
1006  * the one just before that.
1007  */
1008 static struct dwc3_trb *dwc3_ep_prev_trb(struct dwc3_ep *dep, u8 index)
1009 {
1010         u8 tmp = index;
1011
1012         if (!tmp)
1013                 tmp = DWC3_TRB_NUM - 1;
1014
1015         return &dep->trb_pool[tmp - 1];
1016 }
1017
1018 static u32 dwc3_calc_trbs_left(struct dwc3_ep *dep)
1019 {
1020         struct dwc3_trb         *tmp;
1021         u8                      trbs_left;
1022
1023         /*
1024          * If enqueue & dequeue are equal than it is either full or empty.
1025          *
1026          * One way to know for sure is if the TRB right before us has HWO bit
1027          * set or not. If it has, then we're definitely full and can't fit any
1028          * more transfers in our ring.
1029          */
1030         if (dep->trb_enqueue == dep->trb_dequeue) {
1031                 tmp = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1032                 if (tmp->ctrl & DWC3_TRB_CTRL_HWO)
1033                         return 0;
1034
1035                 return DWC3_TRB_NUM - 1;
1036         }
1037
1038         trbs_left = dep->trb_dequeue - dep->trb_enqueue;
1039         trbs_left &= (DWC3_TRB_NUM - 1);
1040
1041         if (dep->trb_dequeue < dep->trb_enqueue)
1042                 trbs_left--;
1043
1044         return trbs_left;
1045 }
1046
1047 static void dwc3_prepare_one_trb_sg(struct dwc3_ep *dep,
1048                 struct dwc3_request *req)
1049 {
1050         struct scatterlist *sg = req->start_sg;
1051         struct scatterlist *s;
1052         int             i;
1053
1054         unsigned int remaining = req->request.num_mapped_sgs
1055                 - req->num_queued_sgs;
1056
1057         for_each_sg(sg, s, remaining, i) {
1058                 unsigned int length = req->request.length;
1059                 unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1060                 unsigned int rem = length % maxp;
1061                 unsigned chain = true;
1062
1063                 if (sg_is_last(s))
1064                         chain = false;
1065
1066                 if (rem && usb_endpoint_dir_out(dep->endpoint.desc) && !chain) {
1067                         struct dwc3     *dwc = dep->dwc;
1068                         struct dwc3_trb *trb;
1069
1070                         req->unaligned = true;
1071
1072                         /* prepare normal TRB */
1073                         dwc3_prepare_one_trb(dep, req, true, i);
1074
1075                         /* Now prepare one extra TRB to align transfer size */
1076                         trb = &dep->trb_pool[dep->trb_enqueue];
1077                         __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr,
1078                                         maxp - rem, false, 0,
1079                                         req->request.stream_id,
1080                                         req->request.short_not_ok,
1081                                         req->request.no_interrupt);
1082                 } else {
1083                         dwc3_prepare_one_trb(dep, req, chain, i);
1084                 }
1085
1086                 /*
1087                  * There can be a situation where all sgs in sglist are not
1088                  * queued because of insufficient trb number. To handle this
1089                  * case, update start_sg to next sg to be queued, so that
1090                  * we have free trbs we can continue queuing from where we
1091                  * previously stopped
1092                  */
1093                 if (chain)
1094                         req->start_sg = sg_next(s);
1095
1096                 req->num_queued_sgs++;
1097
1098                 if (!dwc3_calc_trbs_left(dep))
1099                         break;
1100         }
1101 }
1102
1103 static void dwc3_prepare_one_trb_linear(struct dwc3_ep *dep,
1104                 struct dwc3_request *req)
1105 {
1106         unsigned int length = req->request.length;
1107         unsigned int maxp = usb_endpoint_maxp(dep->endpoint.desc);
1108         unsigned int rem = length % maxp;
1109
1110         if (rem && usb_endpoint_dir_out(dep->endpoint.desc)) {
1111                 struct dwc3     *dwc = dep->dwc;
1112                 struct dwc3_trb *trb;
1113
1114                 req->unaligned = true;
1115
1116                 /* prepare normal TRB */
1117                 dwc3_prepare_one_trb(dep, req, true, 0);
1118
1119                 /* Now prepare one extra TRB to align transfer size */
1120                 trb = &dep->trb_pool[dep->trb_enqueue];
1121                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, maxp - rem,
1122                                 false, 0, req->request.stream_id,
1123                                 req->request.short_not_ok,
1124                                 req->request.no_interrupt);
1125         } else if (req->request.zero && req->request.length &&
1126                    (IS_ALIGNED(req->request.length,dep->endpoint.maxpacket))) {
1127                 struct dwc3     *dwc = dep->dwc;
1128                 struct dwc3_trb *trb;
1129
1130                 req->zero = true;
1131
1132                 /* prepare normal TRB */
1133                 dwc3_prepare_one_trb(dep, req, true, 0);
1134
1135                 /* Now prepare one extra TRB to handle ZLP */
1136                 trb = &dep->trb_pool[dep->trb_enqueue];
1137                 __dwc3_prepare_one_trb(dep, trb, dwc->bounce_addr, 0,
1138                                 false, 0, req->request.stream_id,
1139                                 req->request.short_not_ok,
1140                                 req->request.no_interrupt);
1141         } else {
1142                 dwc3_prepare_one_trb(dep, req, false, 0);
1143         }
1144 }
1145
1146 /*
1147  * dwc3_prepare_trbs - setup TRBs from requests
1148  * @dep: endpoint for which requests are being prepared
1149  *
1150  * The function goes through the requests list and sets up TRBs for the
1151  * transfers. The function returns once there are no more TRBs available or
1152  * it runs out of requests.
1153  */
1154 static void dwc3_prepare_trbs(struct dwc3_ep *dep)
1155 {
1156         struct dwc3_request     *req, *n;
1157
1158         BUILD_BUG_ON_NOT_POWER_OF_2(DWC3_TRB_NUM);
1159
1160         /*
1161          * We can get in a situation where there's a request in the started list
1162          * but there weren't enough TRBs to fully kick it in the first time
1163          * around, so it has been waiting for more TRBs to be freed up.
1164          *
1165          * In that case, we should check if we have a request with pending_sgs
1166          * in the started list and prepare TRBs for that request first,
1167          * otherwise we will prepare TRBs completely out of order and that will
1168          * break things.
1169          */
1170         list_for_each_entry(req, &dep->started_list, list) {
1171                 if (req->num_pending_sgs > 0)
1172                         dwc3_prepare_one_trb_sg(dep, req);
1173
1174                 if (!dwc3_calc_trbs_left(dep))
1175                         return;
1176         }
1177
1178         list_for_each_entry_safe(req, n, &dep->pending_list, list) {
1179                 struct dwc3     *dwc = dep->dwc;
1180                 int             ret;
1181
1182                 ret = usb_gadget_map_request_by_dev(dwc->sysdev, &req->request,
1183                                                     dep->direction);
1184                 if (ret)
1185                         return;
1186
1187                 req->sg                 = req->request.sg;
1188                 req->start_sg           = req->sg;
1189                 req->num_queued_sgs     = 0;
1190                 req->num_pending_sgs    = req->request.num_mapped_sgs;
1191
1192                 if (req->num_pending_sgs > 0)
1193                         dwc3_prepare_one_trb_sg(dep, req);
1194                 else
1195                         dwc3_prepare_one_trb_linear(dep, req);
1196
1197                 if (!dwc3_calc_trbs_left(dep))
1198                         return;
1199         }
1200 }
1201
1202 static int __dwc3_gadget_kick_transfer(struct dwc3_ep *dep)
1203 {
1204         struct dwc3_gadget_ep_cmd_params params;
1205         struct dwc3_request             *req;
1206         int                             starting;
1207         int                             ret;
1208         u32                             cmd;
1209
1210         if (!dwc3_calc_trbs_left(dep))
1211                 return 0;
1212
1213         starting = !(dep->flags & DWC3_EP_TRANSFER_STARTED);
1214
1215         dwc3_prepare_trbs(dep);
1216         req = next_request(&dep->started_list);
1217         if (!req) {
1218                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1219                 return 0;
1220         }
1221
1222         memset(&params, 0, sizeof(params));
1223
1224         if (starting) {
1225                 params.param0 = upper_32_bits(req->trb_dma);
1226                 params.param1 = lower_32_bits(req->trb_dma);
1227                 cmd = DWC3_DEPCMD_STARTTRANSFER;
1228
1229                 if (usb_endpoint_xfer_isoc(dep->endpoint.desc))
1230                         cmd |= DWC3_DEPCMD_PARAM(dep->frame_number);
1231         } else {
1232                 cmd = DWC3_DEPCMD_UPDATETRANSFER |
1233                         DWC3_DEPCMD_PARAM(dep->resource_index);
1234         }
1235
1236         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
1237         if (ret < 0) {
1238                 /*
1239                  * FIXME we need to iterate over the list of requests
1240                  * here and stop, unmap, free and del each of the linked
1241                  * requests instead of what we do now.
1242                  */
1243                 if (req->trb)
1244                         memset(req->trb, 0, sizeof(struct dwc3_trb));
1245                 dwc3_gadget_del_and_unmap_request(dep, req, ret);
1246                 return ret;
1247         }
1248
1249         if (starting) {
1250                 dep->resource_index = dwc3_gadget_ep_get_transfer_index(dep);
1251                 WARN_ON_ONCE(!dep->resource_index);
1252         }
1253
1254         return 0;
1255 }
1256
1257 static int __dwc3_gadget_get_frame(struct dwc3 *dwc)
1258 {
1259         u32                     reg;
1260
1261         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1262         return DWC3_DSTS_SOFFN(reg);
1263 }
1264
1265 static void __dwc3_gadget_start_isoc(struct dwc3_ep *dep)
1266 {
1267         if (list_empty(&dep->pending_list)) {
1268                 dev_info(dep->dwc->dev, "%s: ran out of requests\n",
1269                                 dep->name);
1270                 dep->flags |= DWC3_EP_PENDING_REQUEST;
1271                 return;
1272         }
1273
1274         /*
1275          * Schedule the first trb for one interval in the future or at
1276          * least 4 microframes.
1277          */
1278         dep->frame_number += max_t(u32, 4, dep->interval);
1279         __dwc3_gadget_kick_transfer(dep);
1280 }
1281
1282 static int __dwc3_gadget_ep_queue(struct dwc3_ep *dep, struct dwc3_request *req)
1283 {
1284         struct dwc3             *dwc = dep->dwc;
1285
1286         if (!dep->endpoint.desc) {
1287                 dev_err(dwc->dev, "%s: can't queue to disabled endpoint\n",
1288                                 dep->name);
1289                 return -ESHUTDOWN;
1290         }
1291
1292         if (WARN(req->dep != dep, "request %pK belongs to '%s'\n",
1293                                 &req->request, req->dep->name))
1294                 return -EINVAL;
1295
1296         pm_runtime_get(dwc->dev);
1297
1298         req->request.actual     = 0;
1299         req->request.status     = -EINPROGRESS;
1300         req->direction          = dep->direction;
1301         req->epnum              = dep->number;
1302
1303         trace_dwc3_ep_queue(req);
1304
1305         list_add_tail(&req->list, &dep->pending_list);
1306
1307         /*
1308          * NOTICE: Isochronous endpoints should NEVER be prestarted. We must
1309          * wait for a XferNotReady event so we will know what's the current
1310          * (micro-)frame number.
1311          *
1312          * Without this trick, we are very, very likely gonna get Bus Expiry
1313          * errors which will force us issue EndTransfer command.
1314          */
1315         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1316                 if (!(dep->flags & DWC3_EP_PENDING_REQUEST) &&
1317                                 !(dep->flags & DWC3_EP_TRANSFER_STARTED))
1318                         return 0;
1319
1320                 if ((dep->flags & DWC3_EP_PENDING_REQUEST)) {
1321                         if (!(dep->flags & DWC3_EP_TRANSFER_STARTED)) {
1322                                 __dwc3_gadget_start_isoc(dep);
1323                                 return 0;
1324                         }
1325                 }
1326         }
1327
1328         return __dwc3_gadget_kick_transfer(dep);
1329 }
1330
1331 static int dwc3_gadget_ep_queue(struct usb_ep *ep, struct usb_request *request,
1332         gfp_t gfp_flags)
1333 {
1334         struct dwc3_request             *req = to_dwc3_request(request);
1335         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1336         struct dwc3                     *dwc = dep->dwc;
1337
1338         unsigned long                   flags;
1339
1340         int                             ret;
1341
1342         spin_lock_irqsave(&dwc->lock, flags);
1343         ret = __dwc3_gadget_ep_queue(dep, req);
1344         spin_unlock_irqrestore(&dwc->lock, flags);
1345
1346         return ret;
1347 }
1348
1349 static int dwc3_gadget_ep_dequeue(struct usb_ep *ep,
1350                 struct usb_request *request)
1351 {
1352         struct dwc3_request             *req = to_dwc3_request(request);
1353         struct dwc3_request             *r = NULL;
1354
1355         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1356         struct dwc3                     *dwc = dep->dwc;
1357
1358         unsigned long                   flags;
1359         int                             ret = 0;
1360
1361         trace_dwc3_ep_dequeue(req);
1362
1363         spin_lock_irqsave(&dwc->lock, flags);
1364
1365         list_for_each_entry(r, &dep->pending_list, list) {
1366                 if (r == req)
1367                         break;
1368         }
1369
1370         if (r != req) {
1371                 list_for_each_entry(r, &dep->started_list, list) {
1372                         if (r == req)
1373                                 break;
1374                 }
1375                 if (r == req) {
1376                         /* wait until it is processed */
1377                         dwc3_stop_active_transfer(dep, true);
1378
1379                         /*
1380                          * If request was already started, this means we had to
1381                          * stop the transfer. With that we also need to ignore
1382                          * all TRBs used by the request, however TRBs can only
1383                          * be modified after completion of END_TRANSFER
1384                          * command. So what we do here is that we wait for
1385                          * END_TRANSFER completion and only after that, we jump
1386                          * over TRBs by clearing HWO and incrementing dequeue
1387                          * pointer.
1388                          *
1389                          * Note that we have 2 possible types of transfers here:
1390                          *
1391                          * i) Linear buffer request
1392                          * ii) SG-list based request
1393                          *
1394                          * SG-list based requests will have r->num_pending_sgs
1395                          * set to a valid number (> 0). Linear requests,
1396                          * normally use a single TRB.
1397                          *
1398                          * For each of these two cases, if r->unaligned flag is
1399                          * set, one extra TRB has been used to align transfer
1400                          * size to wMaxPacketSize.
1401                          *
1402                          * All of these cases need to be taken into
1403                          * consideration so we don't mess up our TRB ring
1404                          * pointers.
1405                          */
1406                         wait_event_lock_irq(dep->wait_end_transfer,
1407                                         !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1408                                         dwc->lock);
1409
1410                         if (!r->trb)
1411                                 goto out1;
1412
1413                         if (r->num_pending_sgs) {
1414                                 struct dwc3_trb *trb;
1415                                 int i = 0;
1416
1417                                 for (i = 0; i < r->num_pending_sgs; i++) {
1418                                         trb = r->trb + i;
1419                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1420                                         dwc3_ep_inc_deq(dep);
1421                                 }
1422
1423                                 if (r->unaligned || r->zero) {
1424                                         trb = r->trb + r->num_pending_sgs + 1;
1425                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1426                                         dwc3_ep_inc_deq(dep);
1427                                 }
1428                         } else {
1429                                 struct dwc3_trb *trb = r->trb;
1430
1431                                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1432                                 dwc3_ep_inc_deq(dep);
1433
1434                                 if (r->unaligned || r->zero) {
1435                                         trb = r->trb + 1;
1436                                         trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
1437                                         dwc3_ep_inc_deq(dep);
1438                                 }
1439                         }
1440                         goto out1;
1441                 }
1442                 dev_err(dwc->dev, "request %pK was not queued to %s\n",
1443                                 request, ep->name);
1444                 ret = -EINVAL;
1445                 goto out0;
1446         }
1447
1448 out1:
1449         /* giveback the request */
1450
1451         dwc3_gadget_giveback(dep, req, -ECONNRESET);
1452
1453 out0:
1454         spin_unlock_irqrestore(&dwc->lock, flags);
1455
1456         return ret;
1457 }
1458
1459 int __dwc3_gadget_ep_set_halt(struct dwc3_ep *dep, int value, int protocol)
1460 {
1461         struct dwc3_gadget_ep_cmd_params        params;
1462         struct dwc3                             *dwc = dep->dwc;
1463         int                                     ret;
1464
1465         if (usb_endpoint_xfer_isoc(dep->endpoint.desc)) {
1466                 dev_err(dwc->dev, "%s is of Isochronous type\n", dep->name);
1467                 return -EINVAL;
1468         }
1469
1470         memset(&params, 0x00, sizeof(params));
1471
1472         if (value) {
1473                 struct dwc3_trb *trb;
1474
1475                 unsigned transfer_in_flight;
1476                 unsigned started;
1477
1478                 if (dep->flags & DWC3_EP_STALL)
1479                         return 0;
1480
1481                 if (dep->number > 1)
1482                         trb = dwc3_ep_prev_trb(dep, dep->trb_enqueue);
1483                 else
1484                         trb = &dwc->ep0_trb[dep->trb_enqueue];
1485
1486                 transfer_in_flight = trb->ctrl & DWC3_TRB_CTRL_HWO;
1487                 started = !list_empty(&dep->started_list);
1488
1489                 if (!protocol && ((dep->direction && transfer_in_flight) ||
1490                                 (!dep->direction && started))) {
1491                         return -EAGAIN;
1492                 }
1493
1494                 ret = dwc3_send_gadget_ep_cmd(dep, DWC3_DEPCMD_SETSTALL,
1495                                 &params);
1496                 if (ret)
1497                         dev_err(dwc->dev, "failed to set STALL on %s\n",
1498                                         dep->name);
1499                 else
1500                         dep->flags |= DWC3_EP_STALL;
1501         } else {
1502                 if (!(dep->flags & DWC3_EP_STALL))
1503                         return 0;
1504
1505                 ret = dwc3_send_clear_stall_ep_cmd(dep);
1506                 if (ret)
1507                         dev_err(dwc->dev, "failed to clear STALL on %s\n",
1508                                         dep->name);
1509                 else
1510                         dep->flags &= ~(DWC3_EP_STALL | DWC3_EP_WEDGE);
1511         }
1512
1513         return ret;
1514 }
1515
1516 static int dwc3_gadget_ep_set_halt(struct usb_ep *ep, int value)
1517 {
1518         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1519         struct dwc3                     *dwc = dep->dwc;
1520
1521         unsigned long                   flags;
1522
1523         int                             ret;
1524
1525         spin_lock_irqsave(&dwc->lock, flags);
1526         ret = __dwc3_gadget_ep_set_halt(dep, value, false);
1527         spin_unlock_irqrestore(&dwc->lock, flags);
1528
1529         return ret;
1530 }
1531
1532 static int dwc3_gadget_ep_set_wedge(struct usb_ep *ep)
1533 {
1534         struct dwc3_ep                  *dep = to_dwc3_ep(ep);
1535         struct dwc3                     *dwc = dep->dwc;
1536         unsigned long                   flags;
1537         int                             ret;
1538
1539         spin_lock_irqsave(&dwc->lock, flags);
1540         dep->flags |= DWC3_EP_WEDGE;
1541
1542         if (dep->number == 0 || dep->number == 1)
1543                 ret = __dwc3_gadget_ep0_set_halt(ep, 1);
1544         else
1545                 ret = __dwc3_gadget_ep_set_halt(dep, 1, false);
1546         spin_unlock_irqrestore(&dwc->lock, flags);
1547
1548         return ret;
1549 }
1550
1551 /* -------------------------------------------------------------------------- */
1552
1553 static struct usb_endpoint_descriptor dwc3_gadget_ep0_desc = {
1554         .bLength        = USB_DT_ENDPOINT_SIZE,
1555         .bDescriptorType = USB_DT_ENDPOINT,
1556         .bmAttributes   = USB_ENDPOINT_XFER_CONTROL,
1557 };
1558
1559 static const struct usb_ep_ops dwc3_gadget_ep0_ops = {
1560         .enable         = dwc3_gadget_ep0_enable,
1561         .disable        = dwc3_gadget_ep0_disable,
1562         .alloc_request  = dwc3_gadget_ep_alloc_request,
1563         .free_request   = dwc3_gadget_ep_free_request,
1564         .queue          = dwc3_gadget_ep0_queue,
1565         .dequeue        = dwc3_gadget_ep_dequeue,
1566         .set_halt       = dwc3_gadget_ep0_set_halt,
1567         .set_wedge      = dwc3_gadget_ep_set_wedge,
1568 };
1569
1570 static const struct usb_ep_ops dwc3_gadget_ep_ops = {
1571         .enable         = dwc3_gadget_ep_enable,
1572         .disable        = dwc3_gadget_ep_disable,
1573         .alloc_request  = dwc3_gadget_ep_alloc_request,
1574         .free_request   = dwc3_gadget_ep_free_request,
1575         .queue          = dwc3_gadget_ep_queue,
1576         .dequeue        = dwc3_gadget_ep_dequeue,
1577         .set_halt       = dwc3_gadget_ep_set_halt,
1578         .set_wedge      = dwc3_gadget_ep_set_wedge,
1579 };
1580
1581 /* -------------------------------------------------------------------------- */
1582
1583 static int dwc3_gadget_get_frame(struct usb_gadget *g)
1584 {
1585         struct dwc3             *dwc = gadget_to_dwc(g);
1586
1587         return __dwc3_gadget_get_frame(dwc);
1588 }
1589
1590 static int __dwc3_gadget_wakeup(struct dwc3 *dwc)
1591 {
1592         int                     retries;
1593
1594         int                     ret;
1595         u32                     reg;
1596
1597         u8                      link_state;
1598         u8                      speed;
1599
1600         /*
1601          * According to the Databook Remote wakeup request should
1602          * be issued only when the device is in early suspend state.
1603          *
1604          * We can check that via USB Link State bits in DSTS register.
1605          */
1606         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1607
1608         speed = reg & DWC3_DSTS_CONNECTSPD;
1609         if ((speed == DWC3_DSTS_SUPERSPEED) ||
1610             (speed == DWC3_DSTS_SUPERSPEED_PLUS))
1611                 return 0;
1612
1613         link_state = DWC3_DSTS_USBLNKST(reg);
1614
1615         switch (link_state) {
1616         case DWC3_LINK_STATE_RX_DET:    /* in HS, means Early Suspend */
1617         case DWC3_LINK_STATE_U3:        /* in HS, means SUSPEND */
1618                 break;
1619         default:
1620                 return -EINVAL;
1621         }
1622
1623         ret = dwc3_gadget_set_link_state(dwc, DWC3_LINK_STATE_RECOV);
1624         if (ret < 0) {
1625                 dev_err(dwc->dev, "failed to put link in Recovery\n");
1626                 return ret;
1627         }
1628
1629         /* Recent versions do this automatically */
1630         if (dwc->revision < DWC3_REVISION_194A) {
1631                 /* write zeroes to Link Change Request */
1632                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1633                 reg &= ~DWC3_DCTL_ULSTCHNGREQ_MASK;
1634                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1635         }
1636
1637         /* poll until Link State changes to ON */
1638         retries = 20000;
1639
1640         while (retries--) {
1641                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1642
1643                 /* in HS, means ON */
1644                 if (DWC3_DSTS_USBLNKST(reg) == DWC3_LINK_STATE_U0)
1645                         break;
1646         }
1647
1648         if (DWC3_DSTS_USBLNKST(reg) != DWC3_LINK_STATE_U0) {
1649                 dev_err(dwc->dev, "failed to send remote wakeup\n");
1650                 return -EINVAL;
1651         }
1652
1653         return 0;
1654 }
1655
1656 static int dwc3_gadget_wakeup(struct usb_gadget *g)
1657 {
1658         struct dwc3             *dwc = gadget_to_dwc(g);
1659         unsigned long           flags;
1660         int                     ret;
1661
1662         spin_lock_irqsave(&dwc->lock, flags);
1663         ret = __dwc3_gadget_wakeup(dwc);
1664         spin_unlock_irqrestore(&dwc->lock, flags);
1665
1666         return ret;
1667 }
1668
1669 static int dwc3_gadget_set_selfpowered(struct usb_gadget *g,
1670                 int is_selfpowered)
1671 {
1672         struct dwc3             *dwc = gadget_to_dwc(g);
1673         unsigned long           flags;
1674
1675         spin_lock_irqsave(&dwc->lock, flags);
1676         g->is_selfpowered = !!is_selfpowered;
1677         spin_unlock_irqrestore(&dwc->lock, flags);
1678
1679         return 0;
1680 }
1681
1682 static int dwc3_gadget_run_stop(struct dwc3 *dwc, int is_on, int suspend)
1683 {
1684         u32                     reg;
1685         u32                     timeout = 500;
1686
1687         if (pm_runtime_suspended(dwc->dev))
1688                 return 0;
1689
1690         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
1691         if (is_on) {
1692                 if (dwc->revision <= DWC3_REVISION_187A) {
1693                         reg &= ~DWC3_DCTL_TRGTULST_MASK;
1694                         reg |= DWC3_DCTL_TRGTULST_RX_DET;
1695                 }
1696
1697                 if (dwc->revision >= DWC3_REVISION_194A)
1698                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1699                 reg |= DWC3_DCTL_RUN_STOP;
1700
1701                 if (dwc->has_hibernation)
1702                         reg |= DWC3_DCTL_KEEP_CONNECT;
1703
1704                 dwc->pullups_connected = true;
1705         } else {
1706                 reg &= ~DWC3_DCTL_RUN_STOP;
1707
1708                 if (dwc->has_hibernation && !suspend)
1709                         reg &= ~DWC3_DCTL_KEEP_CONNECT;
1710
1711                 dwc->pullups_connected = false;
1712         }
1713
1714         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
1715
1716         do {
1717                 reg = dwc3_readl(dwc->regs, DWC3_DSTS);
1718                 reg &= DWC3_DSTS_DEVCTRLHLT;
1719         } while (--timeout && !(!is_on ^ !reg));
1720
1721         if (!timeout)
1722                 return -ETIMEDOUT;
1723
1724         return 0;
1725 }
1726
1727 static int dwc3_gadget_pullup(struct usb_gadget *g, int is_on)
1728 {
1729         struct dwc3             *dwc = gadget_to_dwc(g);
1730         unsigned long           flags;
1731         int                     ret;
1732
1733         is_on = !!is_on;
1734
1735         /*
1736          * Per databook, when we want to stop the gadget, if a control transfer
1737          * is still in process, complete it and get the core into setup phase.
1738          */
1739         if (!is_on && dwc->ep0state != EP0_SETUP_PHASE) {
1740                 reinit_completion(&dwc->ep0_in_setup);
1741
1742                 ret = wait_for_completion_timeout(&dwc->ep0_in_setup,
1743                                 msecs_to_jiffies(DWC3_PULL_UP_TIMEOUT));
1744                 if (ret == 0) {
1745                         dev_err(dwc->dev, "timed out waiting for SETUP phase\n");
1746                         return -ETIMEDOUT;
1747                 }
1748         }
1749
1750         spin_lock_irqsave(&dwc->lock, flags);
1751         ret = dwc3_gadget_run_stop(dwc, is_on, false);
1752         spin_unlock_irqrestore(&dwc->lock, flags);
1753
1754         return ret;
1755 }
1756
1757 static void dwc3_gadget_enable_irq(struct dwc3 *dwc)
1758 {
1759         u32                     reg;
1760
1761         /* Enable all but Start and End of Frame IRQs */
1762         reg = (DWC3_DEVTEN_VNDRDEVTSTRCVEDEN |
1763                         DWC3_DEVTEN_EVNTOVERFLOWEN |
1764                         DWC3_DEVTEN_CMDCMPLTEN |
1765                         DWC3_DEVTEN_ERRTICERREN |
1766                         DWC3_DEVTEN_WKUPEVTEN |
1767                         DWC3_DEVTEN_CONNECTDONEEN |
1768                         DWC3_DEVTEN_USBRSTEN |
1769                         DWC3_DEVTEN_DISCONNEVTEN);
1770
1771         if (dwc->revision < DWC3_REVISION_250A)
1772                 reg |= DWC3_DEVTEN_ULSTCNGEN;
1773
1774         dwc3_writel(dwc->regs, DWC3_DEVTEN, reg);
1775 }
1776
1777 static void dwc3_gadget_disable_irq(struct dwc3 *dwc)
1778 {
1779         /* mask all interrupts */
1780         dwc3_writel(dwc->regs, DWC3_DEVTEN, 0x00);
1781 }
1782
1783 static irqreturn_t dwc3_interrupt(int irq, void *_dwc);
1784 static irqreturn_t dwc3_thread_interrupt(int irq, void *_dwc);
1785
1786 /**
1787  * dwc3_gadget_setup_nump - calculate and initialize NUMP field of %DWC3_DCFG
1788  * @dwc: pointer to our context structure
1789  *
1790  * The following looks like complex but it's actually very simple. In order to
1791  * calculate the number of packets we can burst at once on OUT transfers, we're
1792  * gonna use RxFIFO size.
1793  *
1794  * To calculate RxFIFO size we need two numbers:
1795  * MDWIDTH = size, in bits, of the internal memory bus
1796  * RAM2_DEPTH = depth, in MDWIDTH, of internal RAM2 (where RxFIFO sits)
1797  *
1798  * Given these two numbers, the formula is simple:
1799  *
1800  * RxFIFO Size = (RAM2_DEPTH * MDWIDTH / 8) - 24 - 16;
1801  *
1802  * 24 bytes is for 3x SETUP packets
1803  * 16 bytes is a clock domain crossing tolerance
1804  *
1805  * Given RxFIFO Size, NUMP = RxFIFOSize / 1024;
1806  */
1807 static void dwc3_gadget_setup_nump(struct dwc3 *dwc)
1808 {
1809         u32 ram2_depth;
1810         u32 mdwidth;
1811         u32 nump;
1812         u32 reg;
1813
1814         ram2_depth = DWC3_GHWPARAMS7_RAM2_DEPTH(dwc->hwparams.hwparams7);
1815         mdwidth = DWC3_GHWPARAMS0_MDWIDTH(dwc->hwparams.hwparams0);
1816
1817         nump = ((ram2_depth * mdwidth / 8) - 24 - 16) / 1024;
1818         nump = min_t(u32, nump, 16);
1819
1820         /* update NumP */
1821         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
1822         reg &= ~DWC3_DCFG_NUMP_MASK;
1823         reg |= nump << DWC3_DCFG_NUMP_SHIFT;
1824         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
1825 }
1826
1827 static int __dwc3_gadget_start(struct dwc3 *dwc)
1828 {
1829         struct dwc3_ep          *dep;
1830         int                     ret = 0;
1831         u32                     reg;
1832
1833         /*
1834          * Use IMOD if enabled via dwc->imod_interval. Otherwise, if
1835          * the core supports IMOD, disable it.
1836          */
1837         if (dwc->imod_interval) {
1838                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
1839                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
1840         } else if (dwc3_has_imod(dwc)) {
1841                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), 0);
1842         }
1843
1844         /*
1845          * We are telling dwc3 that we want to use DCFG.NUMP as ACK TP's NUMP
1846          * field instead of letting dwc3 itself calculate that automatically.
1847          *
1848          * This way, we maximize the chances that we'll be able to get several
1849          * bursts of data without going through any sort of endpoint throttling.
1850          */
1851         reg = dwc3_readl(dwc->regs, DWC3_GRXTHRCFG);
1852         if (dwc3_is_usb31(dwc))
1853                 reg &= ~DWC31_GRXTHRCFG_PKTCNTSEL;
1854         else
1855                 reg &= ~DWC3_GRXTHRCFG_PKTCNTSEL;
1856
1857         dwc3_writel(dwc->regs, DWC3_GRXTHRCFG, reg);
1858
1859         dwc3_gadget_setup_nump(dwc);
1860
1861         /* Start with SuperSpeed Default */
1862         dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
1863
1864         dep = dwc->eps[0];
1865         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1866         if (ret) {
1867                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1868                 goto err0;
1869         }
1870
1871         dep = dwc->eps[1];
1872         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_INIT);
1873         if (ret) {
1874                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
1875                 goto err1;
1876         }
1877
1878         /* begin to receive SETUP packets */
1879         dwc->ep0state = EP0_SETUP_PHASE;
1880         dwc3_ep0_out_start(dwc);
1881
1882         dwc3_gadget_enable_irq(dwc);
1883
1884         return 0;
1885
1886 err1:
1887         __dwc3_gadget_ep_disable(dwc->eps[0]);
1888
1889 err0:
1890         return ret;
1891 }
1892
1893 static int dwc3_gadget_start(struct usb_gadget *g,
1894                 struct usb_gadget_driver *driver)
1895 {
1896         struct dwc3             *dwc = gadget_to_dwc(g);
1897         unsigned long           flags;
1898         int                     ret = 0;
1899         int                     irq;
1900
1901         irq = dwc->irq_gadget;
1902         ret = request_threaded_irq(irq, dwc3_interrupt, dwc3_thread_interrupt,
1903                         IRQF_SHARED, "dwc3", dwc->ev_buf);
1904         if (ret) {
1905                 dev_err(dwc->dev, "failed to request irq #%d --> %d\n",
1906                                 irq, ret);
1907                 goto err0;
1908         }
1909
1910         spin_lock_irqsave(&dwc->lock, flags);
1911         if (dwc->gadget_driver) {
1912                 dev_err(dwc->dev, "%s is already bound to %s\n",
1913                                 dwc->gadget.name,
1914                                 dwc->gadget_driver->driver.name);
1915                 ret = -EBUSY;
1916                 goto err1;
1917         }
1918
1919         dwc->gadget_driver      = driver;
1920
1921         if (pm_runtime_active(dwc->dev))
1922                 __dwc3_gadget_start(dwc);
1923
1924         spin_unlock_irqrestore(&dwc->lock, flags);
1925
1926         return 0;
1927
1928 err1:
1929         spin_unlock_irqrestore(&dwc->lock, flags);
1930         free_irq(irq, dwc);
1931
1932 err0:
1933         return ret;
1934 }
1935
1936 static void __dwc3_gadget_stop(struct dwc3 *dwc)
1937 {
1938         dwc3_gadget_disable_irq(dwc);
1939         __dwc3_gadget_ep_disable(dwc->eps[0]);
1940         __dwc3_gadget_ep_disable(dwc->eps[1]);
1941 }
1942
1943 static int dwc3_gadget_stop(struct usb_gadget *g)
1944 {
1945         struct dwc3             *dwc = gadget_to_dwc(g);
1946         unsigned long           flags;
1947         int                     epnum;
1948         u32                     tmo_eps = 0;
1949
1950         spin_lock_irqsave(&dwc->lock, flags);
1951
1952         if (pm_runtime_suspended(dwc->dev))
1953                 goto out;
1954
1955         __dwc3_gadget_stop(dwc);
1956
1957         for (epnum = 2; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
1958                 struct dwc3_ep  *dep = dwc->eps[epnum];
1959                 int ret;
1960
1961                 if (!dep)
1962                         continue;
1963
1964                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
1965                         continue;
1966
1967                 ret = wait_event_interruptible_lock_irq_timeout(dep->wait_end_transfer,
1968                             !(dep->flags & DWC3_EP_END_TRANSFER_PENDING),
1969                             dwc->lock, msecs_to_jiffies(5));
1970
1971                 if (ret <= 0) {
1972                         /* Timed out or interrupted! There's nothing much
1973                          * we can do so we just log here and print which
1974                          * endpoints timed out at the end.
1975                          */
1976                         tmo_eps |= 1 << epnum;
1977                         dep->flags &= DWC3_EP_END_TRANSFER_PENDING;
1978                 }
1979         }
1980
1981         if (tmo_eps) {
1982                 dev_err(dwc->dev,
1983                         "end transfer timed out on endpoints 0x%x [bitmap]\n",
1984                         tmo_eps);
1985         }
1986
1987 out:
1988         dwc->gadget_driver      = NULL;
1989         spin_unlock_irqrestore(&dwc->lock, flags);
1990
1991         free_irq(dwc->irq_gadget, dwc->ev_buf);
1992
1993         return 0;
1994 }
1995
1996 static void dwc3_gadget_set_speed(struct usb_gadget *g,
1997                                   enum usb_device_speed speed)
1998 {
1999         struct dwc3             *dwc = gadget_to_dwc(g);
2000         unsigned long           flags;
2001         u32                     reg;
2002
2003         spin_lock_irqsave(&dwc->lock, flags);
2004         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2005         reg &= ~(DWC3_DCFG_SPEED_MASK);
2006
2007         /*
2008          * WORKAROUND: DWC3 revision < 2.20a have an issue
2009          * which would cause metastability state on Run/Stop
2010          * bit if we try to force the IP to USB2-only mode.
2011          *
2012          * Because of that, we cannot configure the IP to any
2013          * speed other than the SuperSpeed
2014          *
2015          * Refers to:
2016          *
2017          * STAR#9000525659: Clock Domain Crossing on DCTL in
2018          * USB 2.0 Mode
2019          */
2020         if (dwc->revision < DWC3_REVISION_220A &&
2021             !dwc->dis_metastability_quirk) {
2022                 reg |= DWC3_DCFG_SUPERSPEED;
2023         } else {
2024                 switch (speed) {
2025                 case USB_SPEED_LOW:
2026                         reg |= DWC3_DCFG_LOWSPEED;
2027                         break;
2028                 case USB_SPEED_FULL:
2029                         reg |= DWC3_DCFG_FULLSPEED;
2030                         break;
2031                 case USB_SPEED_HIGH:
2032                         reg |= DWC3_DCFG_HIGHSPEED;
2033                         break;
2034                 case USB_SPEED_SUPER:
2035                         reg |= DWC3_DCFG_SUPERSPEED;
2036                         break;
2037                 case USB_SPEED_SUPER_PLUS:
2038                         if (dwc3_is_usb31(dwc))
2039                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2040                         else
2041                                 reg |= DWC3_DCFG_SUPERSPEED;
2042                         break;
2043                 default:
2044                         dev_err(dwc->dev, "invalid speed (%d)\n", speed);
2045
2046                         if (dwc->revision & DWC3_REVISION_IS_DWC31)
2047                                 reg |= DWC3_DCFG_SUPERSPEED_PLUS;
2048                         else
2049                                 reg |= DWC3_DCFG_SUPERSPEED;
2050                 }
2051         }
2052         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2053
2054         spin_unlock_irqrestore(&dwc->lock, flags);
2055 }
2056
2057 static const struct usb_gadget_ops dwc3_gadget_ops = {
2058         .get_frame              = dwc3_gadget_get_frame,
2059         .wakeup                 = dwc3_gadget_wakeup,
2060         .set_selfpowered        = dwc3_gadget_set_selfpowered,
2061         .pullup                 = dwc3_gadget_pullup,
2062         .udc_start              = dwc3_gadget_start,
2063         .udc_stop               = dwc3_gadget_stop,
2064         .udc_set_speed          = dwc3_gadget_set_speed,
2065 };
2066
2067 /* -------------------------------------------------------------------------- */
2068
2069 static int dwc3_gadget_init_control_endpoint(struct dwc3_ep *dep)
2070 {
2071         struct dwc3 *dwc = dep->dwc;
2072
2073         usb_ep_set_maxpacket_limit(&dep->endpoint, 512);
2074         dep->endpoint.maxburst = 1;
2075         dep->endpoint.ops = &dwc3_gadget_ep0_ops;
2076         if (!dep->direction)
2077                 dwc->gadget.ep0 = &dep->endpoint;
2078
2079         dep->endpoint.caps.type_control = true;
2080
2081         return 0;
2082 }
2083
2084 static int dwc3_gadget_init_in_endpoint(struct dwc3_ep *dep)
2085 {
2086         struct dwc3 *dwc = dep->dwc;
2087         int mdwidth;
2088         int kbytes;
2089         int size;
2090
2091         mdwidth = DWC3_MDWIDTH(dwc->hwparams.hwparams0);
2092         /* MDWIDTH is represented in bits, we need it in bytes */
2093         mdwidth /= 8;
2094
2095         size = dwc3_readl(dwc->regs, DWC3_GTXFIFOSIZ(dep->number >> 1));
2096         if (dwc3_is_usb31(dwc))
2097                 size = DWC31_GTXFIFOSIZ_TXFDEF(size);
2098         else
2099                 size = DWC3_GTXFIFOSIZ_TXFDEF(size);
2100
2101         /* FIFO Depth is in MDWDITH bytes. Multiply */
2102         size *= mdwidth;
2103
2104         kbytes = size / 1024;
2105         if (kbytes == 0)
2106                 kbytes = 1;
2107
2108         /*
2109          * FIFO sizes account an extra MDWIDTH * (kbytes + 1) bytes for
2110          * internal overhead. We don't really know how these are used,
2111          * but documentation say it exists.
2112          */
2113         size -= mdwidth * (kbytes + 1);
2114         size /= kbytes;
2115
2116         usb_ep_set_maxpacket_limit(&dep->endpoint, size);
2117
2118         dep->endpoint.max_streams = 15;
2119         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2120         list_add_tail(&dep->endpoint.ep_list,
2121                         &dwc->gadget.ep_list);
2122         dep->endpoint.caps.type_iso = true;
2123         dep->endpoint.caps.type_bulk = true;
2124         dep->endpoint.caps.type_int = true;
2125
2126         return dwc3_alloc_trb_pool(dep);
2127 }
2128
2129 static int dwc3_gadget_init_out_endpoint(struct dwc3_ep *dep)
2130 {
2131         struct dwc3 *dwc = dep->dwc;
2132
2133         usb_ep_set_maxpacket_limit(&dep->endpoint, 1024);
2134         dep->endpoint.max_streams = 15;
2135         dep->endpoint.ops = &dwc3_gadget_ep_ops;
2136         list_add_tail(&dep->endpoint.ep_list,
2137                         &dwc->gadget.ep_list);
2138         dep->endpoint.caps.type_iso = true;
2139         dep->endpoint.caps.type_bulk = true;
2140         dep->endpoint.caps.type_int = true;
2141
2142         return dwc3_alloc_trb_pool(dep);
2143 }
2144
2145 static int dwc3_gadget_init_endpoint(struct dwc3 *dwc, u8 epnum)
2146 {
2147         struct dwc3_ep                  *dep;
2148         bool                            direction = epnum & 1;
2149         int                             ret;
2150         u8                              num = epnum >> 1;
2151
2152         dep = kzalloc(sizeof(*dep), GFP_KERNEL);
2153         if (!dep)
2154                 return -ENOMEM;
2155
2156         dep->dwc = dwc;
2157         dep->number = epnum;
2158         dep->direction = direction;
2159         dep->regs = dwc->regs + DWC3_DEP_BASE(epnum);
2160         dwc->eps[epnum] = dep;
2161
2162         snprintf(dep->name, sizeof(dep->name), "ep%u%s", num,
2163                         direction ? "in" : "out");
2164
2165         dep->endpoint.name = dep->name;
2166
2167         if (!(dep->number > 1)) {
2168                 dep->endpoint.desc = &dwc3_gadget_ep0_desc;
2169                 dep->endpoint.comp_desc = NULL;
2170         }
2171
2172         spin_lock_init(&dep->lock);
2173
2174         if (num == 0)
2175                 ret = dwc3_gadget_init_control_endpoint(dep);
2176         else if (direction)
2177                 ret = dwc3_gadget_init_in_endpoint(dep);
2178         else
2179                 ret = dwc3_gadget_init_out_endpoint(dep);
2180
2181         if (ret)
2182                 return ret;
2183
2184         dep->endpoint.caps.dir_in = direction;
2185         dep->endpoint.caps.dir_out = !direction;
2186
2187         INIT_LIST_HEAD(&dep->pending_list);
2188         INIT_LIST_HEAD(&dep->started_list);
2189
2190         return 0;
2191 }
2192
2193 static int dwc3_gadget_init_endpoints(struct dwc3 *dwc, u8 total)
2194 {
2195         u8                              epnum;
2196
2197         INIT_LIST_HEAD(&dwc->gadget.ep_list);
2198
2199         for (epnum = 0; epnum < total; epnum++) {
2200                 int                     ret;
2201
2202                 ret = dwc3_gadget_init_endpoint(dwc, epnum);
2203                 if (ret)
2204                         return ret;
2205         }
2206
2207         return 0;
2208 }
2209
2210 static void dwc3_gadget_free_endpoints(struct dwc3 *dwc)
2211 {
2212         struct dwc3_ep                  *dep;
2213         u8                              epnum;
2214
2215         for (epnum = 0; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2216                 dep = dwc->eps[epnum];
2217                 if (!dep)
2218                         continue;
2219                 /*
2220                  * Physical endpoints 0 and 1 are special; they form the
2221                  * bi-directional USB endpoint 0.
2222                  *
2223                  * For those two physical endpoints, we don't allocate a TRB
2224                  * pool nor do we add them the endpoints list. Due to that, we
2225                  * shouldn't do these two operations otherwise we would end up
2226                  * with all sorts of bugs when removing dwc3.ko.
2227                  */
2228                 if (epnum != 0 && epnum != 1) {
2229                         dwc3_free_trb_pool(dep);
2230                         list_del(&dep->endpoint.ep_list);
2231                 }
2232
2233                 kfree(dep);
2234         }
2235 }
2236
2237 /* -------------------------------------------------------------------------- */
2238
2239 static int dwc3_gadget_ep_reclaim_completed_trb(struct dwc3_ep *dep,
2240                 struct dwc3_request *req, struct dwc3_trb *trb,
2241                 const struct dwc3_event_depevt *event, int status, int chain)
2242 {
2243         unsigned int            count;
2244
2245         dwc3_ep_inc_deq(dep);
2246
2247         trace_dwc3_complete_trb(dep, trb);
2248
2249         /*
2250          * If we're in the middle of series of chained TRBs and we
2251          * receive a short transfer along the way, DWC3 will skip
2252          * through all TRBs including the last TRB in the chain (the
2253          * where CHN bit is zero. DWC3 will also avoid clearing HWO
2254          * bit and SW has to do it manually.
2255          *
2256          * We're going to do that here to avoid problems of HW trying
2257          * to use bogus TRBs for transfers.
2258          */
2259         if (chain && (trb->ctrl & DWC3_TRB_CTRL_HWO))
2260                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2261
2262         /*
2263          * If we're dealing with unaligned size OUT transfer, we will be left
2264          * with one TRB pending in the ring. We need to manually clear HWO bit
2265          * from that TRB.
2266          */
2267         if ((req->zero || req->unaligned) && (trb->ctrl & DWC3_TRB_CTRL_HWO)) {
2268                 trb->ctrl &= ~DWC3_TRB_CTRL_HWO;
2269                 return 1;
2270         }
2271
2272         count = trb->size & DWC3_TRB_SIZE_MASK;
2273         req->remaining += count;
2274
2275         if ((trb->ctrl & DWC3_TRB_CTRL_HWO) && status != -ESHUTDOWN)
2276                 return 1;
2277
2278         if (event->status & DEPEVT_STATUS_SHORT && !chain)
2279                 return 1;
2280
2281         if (event->status & DEPEVT_STATUS_IOC)
2282                 return 1;
2283
2284         return 0;
2285 }
2286
2287 static int dwc3_gadget_ep_reclaim_trb_sg(struct dwc3_ep *dep,
2288                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2289                 int status)
2290 {
2291         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2292         struct scatterlist *sg = req->sg;
2293         struct scatterlist *s;
2294         unsigned int pending = req->num_pending_sgs;
2295         unsigned int i;
2296         int ret = 0;
2297
2298         for_each_sg(sg, s, pending, i) {
2299                 trb = &dep->trb_pool[dep->trb_dequeue];
2300
2301                 if (trb->ctrl & DWC3_TRB_CTRL_HWO)
2302                         break;
2303
2304                 req->sg = sg_next(s);
2305                 req->num_pending_sgs--;
2306
2307                 ret = dwc3_gadget_ep_reclaim_completed_trb(dep, req,
2308                                 trb, event, status, true);
2309                 if (ret)
2310                         break;
2311         }
2312
2313         return ret;
2314 }
2315
2316 static int dwc3_gadget_ep_reclaim_trb_linear(struct dwc3_ep *dep,
2317                 struct dwc3_request *req, const struct dwc3_event_depevt *event,
2318                 int status)
2319 {
2320         struct dwc3_trb *trb = &dep->trb_pool[dep->trb_dequeue];
2321
2322         return dwc3_gadget_ep_reclaim_completed_trb(dep, req, trb,
2323                         event, status, false);
2324 }
2325
2326 static bool dwc3_gadget_ep_request_completed(struct dwc3_request *req)
2327 {
2328         return req->request.actual == req->request.length;
2329 }
2330
2331 static int dwc3_gadget_ep_cleanup_completed_request(struct dwc3_ep *dep,
2332                 const struct dwc3_event_depevt *event,
2333                 struct dwc3_request *req, int status)
2334 {
2335         int ret;
2336
2337         if (req->num_pending_sgs)
2338                 ret = dwc3_gadget_ep_reclaim_trb_sg(dep, req, event,
2339                                 status);
2340         else
2341                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2342                                 status);
2343
2344         if (req->unaligned || req->zero) {
2345                 ret = dwc3_gadget_ep_reclaim_trb_linear(dep, req, event,
2346                                 status);
2347                 req->unaligned = false;
2348                 req->zero = false;
2349         }
2350
2351         req->request.actual = req->request.length - req->remaining;
2352
2353         if (!dwc3_gadget_ep_request_completed(req) &&
2354                         req->num_pending_sgs) {
2355                 __dwc3_gadget_kick_transfer(dep);
2356                 goto out;
2357         }
2358
2359         dwc3_gadget_giveback(dep, req, status);
2360
2361 out:
2362         return ret;
2363 }
2364
2365 static void dwc3_gadget_ep_cleanup_completed_requests(struct dwc3_ep *dep,
2366                 const struct dwc3_event_depevt *event, int status)
2367 {
2368         struct dwc3_request     *req;
2369         struct dwc3_request     *tmp;
2370
2371         list_for_each_entry_safe(req, tmp, &dep->started_list, list) {
2372                 int ret;
2373
2374                 ret = dwc3_gadget_ep_cleanup_completed_request(dep, event,
2375                                 req, status);
2376                 if (ret)
2377                         break;
2378         }
2379 }
2380
2381 static void dwc3_gadget_endpoint_frame_from_event(struct dwc3_ep *dep,
2382                 const struct dwc3_event_depevt *event)
2383 {
2384         u32 cur_uf, mask;
2385
2386         mask = ~(dep->interval - 1);
2387         cur_uf = event->parameters & mask;
2388         dep->frame_number = cur_uf;
2389 }
2390
2391 static void dwc3_gadget_endpoint_transfer_in_progress(struct dwc3_ep *dep,
2392                 const struct dwc3_event_depevt *event)
2393 {
2394         struct dwc3             *dwc = dep->dwc;
2395         unsigned                status = 0;
2396         bool                    stop = false;
2397
2398         dwc3_gadget_endpoint_frame_from_event(dep, event);
2399
2400         if (event->status & DEPEVT_STATUS_BUSERR)
2401                 status = -ECONNRESET;
2402
2403         if (event->status & DEPEVT_STATUS_MISSED_ISOC) {
2404                 status = -EXDEV;
2405                 stop = true;
2406         }
2407
2408         dwc3_gadget_ep_cleanup_completed_requests(dep, event, status);
2409
2410         if (stop) {
2411                 dwc3_stop_active_transfer(dep, true);
2412                 dep->flags = DWC3_EP_ENABLED;
2413         }
2414
2415         /*
2416          * WORKAROUND: This is the 2nd half of U1/U2 -> U0 workaround.
2417          * See dwc3_gadget_linksts_change_interrupt() for 1st half.
2418          */
2419         if (dwc->revision < DWC3_REVISION_183A) {
2420                 u32             reg;
2421                 int             i;
2422
2423                 for (i = 0; i < DWC3_ENDPOINTS_NUM; i++) {
2424                         dep = dwc->eps[i];
2425
2426                         if (!(dep->flags & DWC3_EP_ENABLED))
2427                                 continue;
2428
2429                         if (!list_empty(&dep->started_list))
2430                                 return;
2431                 }
2432
2433                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2434                 reg |= dwc->u1u2;
2435                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2436
2437                 dwc->u1u2 = 0;
2438         }
2439 }
2440
2441 static void dwc3_gadget_endpoint_transfer_not_ready(struct dwc3_ep *dep,
2442                 const struct dwc3_event_depevt *event)
2443 {
2444         dwc3_gadget_endpoint_frame_from_event(dep, event);
2445         __dwc3_gadget_start_isoc(dep);
2446 }
2447
2448 static void dwc3_endpoint_interrupt(struct dwc3 *dwc,
2449                 const struct dwc3_event_depevt *event)
2450 {
2451         struct dwc3_ep          *dep;
2452         u8                      epnum = event->endpoint_number;
2453         u8                      cmd;
2454
2455         dep = dwc->eps[epnum];
2456
2457         if (!(dep->flags & DWC3_EP_ENABLED)) {
2458                 if (!(dep->flags & DWC3_EP_END_TRANSFER_PENDING))
2459                         return;
2460
2461                 /* Handle only EPCMDCMPLT when EP disabled */
2462                 if (event->endpoint_event != DWC3_DEPEVT_EPCMDCMPLT)
2463                         return;
2464         }
2465
2466         if (epnum == 0 || epnum == 1) {
2467                 dwc3_ep0_interrupt(dwc, event);
2468                 return;
2469         }
2470
2471         switch (event->endpoint_event) {
2472         case DWC3_DEPEVT_XFERINPROGRESS:
2473                 dwc3_gadget_endpoint_transfer_in_progress(dep, event);
2474                 break;
2475         case DWC3_DEPEVT_XFERNOTREADY:
2476                 dwc3_gadget_endpoint_transfer_not_ready(dep, event);
2477                 break;
2478         case DWC3_DEPEVT_EPCMDCMPLT:
2479                 cmd = DEPEVT_PARAMETER_CMD(event->parameters);
2480
2481                 if (cmd == DWC3_DEPCMD_ENDTRANSFER) {
2482                         dep->flags &= ~DWC3_EP_END_TRANSFER_PENDING;
2483                         wake_up(&dep->wait_end_transfer);
2484                 }
2485                 break;
2486         case DWC3_DEPEVT_STREAMEVT:
2487         case DWC3_DEPEVT_XFERCOMPLETE:
2488         case DWC3_DEPEVT_RXTXFIFOEVT:
2489                 break;
2490         }
2491 }
2492
2493 static void dwc3_disconnect_gadget(struct dwc3 *dwc)
2494 {
2495         if (dwc->gadget_driver && dwc->gadget_driver->disconnect) {
2496                 spin_unlock(&dwc->lock);
2497                 dwc->gadget_driver->disconnect(&dwc->gadget);
2498                 spin_lock(&dwc->lock);
2499         }
2500 }
2501
2502 static void dwc3_suspend_gadget(struct dwc3 *dwc)
2503 {
2504         if (dwc->gadget_driver && dwc->gadget_driver->suspend) {
2505                 spin_unlock(&dwc->lock);
2506                 dwc->gadget_driver->suspend(&dwc->gadget);
2507                 spin_lock(&dwc->lock);
2508         }
2509 }
2510
2511 static void dwc3_resume_gadget(struct dwc3 *dwc)
2512 {
2513         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2514                 spin_unlock(&dwc->lock);
2515                 dwc->gadget_driver->resume(&dwc->gadget);
2516                 spin_lock(&dwc->lock);
2517         }
2518 }
2519
2520 static void dwc3_reset_gadget(struct dwc3 *dwc)
2521 {
2522         if (!dwc->gadget_driver)
2523                 return;
2524
2525         if (dwc->gadget.speed != USB_SPEED_UNKNOWN) {
2526                 spin_unlock(&dwc->lock);
2527                 usb_gadget_udc_reset(&dwc->gadget, dwc->gadget_driver);
2528                 spin_lock(&dwc->lock);
2529         }
2530 }
2531
2532 static void dwc3_stop_active_transfer(struct dwc3_ep *dep, bool force)
2533 {
2534         struct dwc3 *dwc = dep->dwc;
2535         struct dwc3_gadget_ep_cmd_params params;
2536         u32 cmd;
2537         int ret;
2538
2539         if ((dep->flags & DWC3_EP_END_TRANSFER_PENDING) ||
2540             !dep->resource_index)
2541                 return;
2542
2543         /*
2544          * NOTICE: We are violating what the Databook says about the
2545          * EndTransfer command. Ideally we would _always_ wait for the
2546          * EndTransfer Command Completion IRQ, but that's causing too
2547          * much trouble synchronizing between us and gadget driver.
2548          *
2549          * We have discussed this with the IP Provider and it was
2550          * suggested to giveback all requests here, but give HW some
2551          * extra time to synchronize with the interconnect. We're using
2552          * an arbitrary 100us delay for that.
2553          *
2554          * Note also that a similar handling was tested by Synopsys
2555          * (thanks a lot Paul) and nothing bad has come out of it.
2556          * In short, what we're doing is:
2557          *
2558          * - Issue EndTransfer WITH CMDIOC bit set
2559          * - Wait 100us
2560          *
2561          * As of IP version 3.10a of the DWC_usb3 IP, the controller
2562          * supports a mode to work around the above limitation. The
2563          * software can poll the CMDACT bit in the DEPCMD register
2564          * after issuing a EndTransfer command. This mode is enabled
2565          * by writing GUCTL2[14]. This polling is already done in the
2566          * dwc3_send_gadget_ep_cmd() function so if the mode is
2567          * enabled, the EndTransfer command will have completed upon
2568          * returning from this function and we don't need to delay for
2569          * 100us.
2570          *
2571          * This mode is NOT available on the DWC_usb31 IP.
2572          */
2573
2574         cmd = DWC3_DEPCMD_ENDTRANSFER;
2575         cmd |= force ? DWC3_DEPCMD_HIPRI_FORCERM : 0;
2576         cmd |= DWC3_DEPCMD_CMDIOC;
2577         cmd |= DWC3_DEPCMD_PARAM(dep->resource_index);
2578         memset(&params, 0, sizeof(params));
2579         ret = dwc3_send_gadget_ep_cmd(dep, cmd, &params);
2580         WARN_ON_ONCE(ret);
2581         dep->resource_index = 0;
2582
2583         if (dwc3_is_usb31(dwc) || dwc->revision < DWC3_REVISION_310A) {
2584                 dep->flags |= DWC3_EP_END_TRANSFER_PENDING;
2585                 udelay(100);
2586         }
2587 }
2588
2589 static void dwc3_clear_stall_all_ep(struct dwc3 *dwc)
2590 {
2591         u32 epnum;
2592
2593         for (epnum = 1; epnum < DWC3_ENDPOINTS_NUM; epnum++) {
2594                 struct dwc3_ep *dep;
2595                 int ret;
2596
2597                 dep = dwc->eps[epnum];
2598                 if (!dep)
2599                         continue;
2600
2601                 if (!(dep->flags & DWC3_EP_STALL))
2602                         continue;
2603
2604                 dep->flags &= ~DWC3_EP_STALL;
2605
2606                 ret = dwc3_send_clear_stall_ep_cmd(dep);
2607                 WARN_ON_ONCE(ret);
2608         }
2609 }
2610
2611 static void dwc3_gadget_disconnect_interrupt(struct dwc3 *dwc)
2612 {
2613         int                     reg;
2614
2615         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2616         reg &= ~DWC3_DCTL_INITU1ENA;
2617         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2618
2619         reg &= ~DWC3_DCTL_INITU2ENA;
2620         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2621
2622         dwc3_disconnect_gadget(dwc);
2623
2624         dwc->gadget.speed = USB_SPEED_UNKNOWN;
2625         dwc->setup_packet_pending = false;
2626         usb_gadget_set_state(&dwc->gadget, USB_STATE_NOTATTACHED);
2627
2628         dwc->connected = false;
2629 }
2630
2631 static void dwc3_gadget_reset_interrupt(struct dwc3 *dwc)
2632 {
2633         u32                     reg;
2634
2635         dwc->connected = true;
2636
2637         /*
2638          * WORKAROUND: DWC3 revisions <1.88a have an issue which
2639          * would cause a missing Disconnect Event if there's a
2640          * pending Setup Packet in the FIFO.
2641          *
2642          * There's no suggested workaround on the official Bug
2643          * report, which states that "unless the driver/application
2644          * is doing any special handling of a disconnect event,
2645          * there is no functional issue".
2646          *
2647          * Unfortunately, it turns out that we _do_ some special
2648          * handling of a disconnect event, namely complete all
2649          * pending transfers, notify gadget driver of the
2650          * disconnection, and so on.
2651          *
2652          * Our suggested workaround is to follow the Disconnect
2653          * Event steps here, instead, based on a setup_packet_pending
2654          * flag. Such flag gets set whenever we have a SETUP_PENDING
2655          * status for EP0 TRBs and gets cleared on XferComplete for the
2656          * same endpoint.
2657          *
2658          * Refers to:
2659          *
2660          * STAR#9000466709: RTL: Device : Disconnect event not
2661          * generated if setup packet pending in FIFO
2662          */
2663         if (dwc->revision < DWC3_REVISION_188A) {
2664                 if (dwc->setup_packet_pending)
2665                         dwc3_gadget_disconnect_interrupt(dwc);
2666         }
2667
2668         dwc3_reset_gadget(dwc);
2669
2670         reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2671         reg &= ~DWC3_DCTL_TSTCTRL_MASK;
2672         dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2673         dwc->test_mode = false;
2674         dwc3_clear_stall_all_ep(dwc);
2675
2676         /* Reset device address to zero */
2677         reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2678         reg &= ~(DWC3_DCFG_DEVADDR_MASK);
2679         dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2680 }
2681
2682 static void dwc3_gadget_conndone_interrupt(struct dwc3 *dwc)
2683 {
2684         struct dwc3_ep          *dep;
2685         int                     ret;
2686         u32                     reg;
2687         u8                      speed;
2688
2689         reg = dwc3_readl(dwc->regs, DWC3_DSTS);
2690         speed = reg & DWC3_DSTS_CONNECTSPD;
2691         dwc->speed = speed;
2692
2693         /*
2694          * RAMClkSel is reset to 0 after USB reset, so it must be reprogrammed
2695          * each time on Connect Done.
2696          *
2697          * Currently we always use the reset value. If any platform
2698          * wants to set this to a different value, we need to add a
2699          * setting and update GCTL.RAMCLKSEL here.
2700          */
2701
2702         switch (speed) {
2703         case DWC3_DSTS_SUPERSPEED_PLUS:
2704                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2705                 dwc->gadget.ep0->maxpacket = 512;
2706                 dwc->gadget.speed = USB_SPEED_SUPER_PLUS;
2707                 break;
2708         case DWC3_DSTS_SUPERSPEED:
2709                 /*
2710                  * WORKAROUND: DWC3 revisions <1.90a have an issue which
2711                  * would cause a missing USB3 Reset event.
2712                  *
2713                  * In such situations, we should force a USB3 Reset
2714                  * event by calling our dwc3_gadget_reset_interrupt()
2715                  * routine.
2716                  *
2717                  * Refers to:
2718                  *
2719                  * STAR#9000483510: RTL: SS : USB3 reset event may
2720                  * not be generated always when the link enters poll
2721                  */
2722                 if (dwc->revision < DWC3_REVISION_190A)
2723                         dwc3_gadget_reset_interrupt(dwc);
2724
2725                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(512);
2726                 dwc->gadget.ep0->maxpacket = 512;
2727                 dwc->gadget.speed = USB_SPEED_SUPER;
2728                 break;
2729         case DWC3_DSTS_HIGHSPEED:
2730                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2731                 dwc->gadget.ep0->maxpacket = 64;
2732                 dwc->gadget.speed = USB_SPEED_HIGH;
2733                 break;
2734         case DWC3_DSTS_FULLSPEED:
2735                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(64);
2736                 dwc->gadget.ep0->maxpacket = 64;
2737                 dwc->gadget.speed = USB_SPEED_FULL;
2738                 break;
2739         case DWC3_DSTS_LOWSPEED:
2740                 dwc3_gadget_ep0_desc.wMaxPacketSize = cpu_to_le16(8);
2741                 dwc->gadget.ep0->maxpacket = 8;
2742                 dwc->gadget.speed = USB_SPEED_LOW;
2743                 break;
2744         }
2745
2746         dwc->eps[1]->endpoint.maxpacket = dwc->gadget.ep0->maxpacket;
2747
2748         /* Enable USB2 LPM Capability */
2749
2750         if ((dwc->revision > DWC3_REVISION_194A) &&
2751             (speed != DWC3_DSTS_SUPERSPEED) &&
2752             (speed != DWC3_DSTS_SUPERSPEED_PLUS)) {
2753                 reg = dwc3_readl(dwc->regs, DWC3_DCFG);
2754                 reg |= DWC3_DCFG_LPM_CAP;
2755                 dwc3_writel(dwc->regs, DWC3_DCFG, reg);
2756
2757                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2758                 reg &= ~(DWC3_DCTL_HIRD_THRES_MASK | DWC3_DCTL_L1_HIBER_EN);
2759
2760                 reg |= DWC3_DCTL_HIRD_THRES(dwc->hird_threshold);
2761
2762                 /*
2763                  * When dwc3 revisions >= 2.40a, LPM Erratum is enabled and
2764                  * DCFG.LPMCap is set, core responses with an ACK and the
2765                  * BESL value in the LPM token is less than or equal to LPM
2766                  * NYET threshold.
2767                  */
2768                 WARN_ONCE(dwc->revision < DWC3_REVISION_240A
2769                                 && dwc->has_lpm_erratum,
2770                                 "LPM Erratum not available on dwc3 revisions < 2.40a\n");
2771
2772                 if (dwc->has_lpm_erratum && dwc->revision >= DWC3_REVISION_240A)
2773                         reg |= DWC3_DCTL_LPM_ERRATA(dwc->lpm_nyet_threshold);
2774
2775                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2776         } else {
2777                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2778                 reg &= ~DWC3_DCTL_HIRD_THRES_MASK;
2779                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2780         }
2781
2782         dep = dwc->eps[0];
2783         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2784         if (ret) {
2785                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2786                 return;
2787         }
2788
2789         dep = dwc->eps[1];
2790         ret = __dwc3_gadget_ep_enable(dep, DWC3_DEPCFG_ACTION_MODIFY);
2791         if (ret) {
2792                 dev_err(dwc->dev, "failed to enable %s\n", dep->name);
2793                 return;
2794         }
2795
2796         /*
2797          * Configure PHY via GUSB3PIPECTLn if required.
2798          *
2799          * Update GTXFIFOSIZn
2800          *
2801          * In both cases reset values should be sufficient.
2802          */
2803 }
2804
2805 static void dwc3_gadget_wakeup_interrupt(struct dwc3 *dwc)
2806 {
2807         /*
2808          * TODO take core out of low power mode when that's
2809          * implemented.
2810          */
2811
2812         if (dwc->gadget_driver && dwc->gadget_driver->resume) {
2813                 spin_unlock(&dwc->lock);
2814                 dwc->gadget_driver->resume(&dwc->gadget);
2815                 spin_lock(&dwc->lock);
2816         }
2817 }
2818
2819 static void dwc3_gadget_linksts_change_interrupt(struct dwc3 *dwc,
2820                 unsigned int evtinfo)
2821 {
2822         enum dwc3_link_state    next = evtinfo & DWC3_LINK_STATE_MASK;
2823         unsigned int            pwropt;
2824
2825         /*
2826          * WORKAROUND: DWC3 < 2.50a have an issue when configured without
2827          * Hibernation mode enabled which would show up when device detects
2828          * host-initiated U3 exit.
2829          *
2830          * In that case, device will generate a Link State Change Interrupt
2831          * from U3 to RESUME which is only necessary if Hibernation is
2832          * configured in.
2833          *
2834          * There are no functional changes due to such spurious event and we
2835          * just need to ignore it.
2836          *
2837          * Refers to:
2838          *
2839          * STAR#9000570034 RTL: SS Resume event generated in non-Hibernation
2840          * operational mode
2841          */
2842         pwropt = DWC3_GHWPARAMS1_EN_PWROPT(dwc->hwparams.hwparams1);
2843         if ((dwc->revision < DWC3_REVISION_250A) &&
2844                         (pwropt != DWC3_GHWPARAMS1_EN_PWROPT_HIB)) {
2845                 if ((dwc->link_state == DWC3_LINK_STATE_U3) &&
2846                                 (next == DWC3_LINK_STATE_RESUME)) {
2847                         return;
2848                 }
2849         }
2850
2851         /*
2852          * WORKAROUND: DWC3 Revisions <1.83a have an issue which, depending
2853          * on the link partner, the USB session might do multiple entry/exit
2854          * of low power states before a transfer takes place.
2855          *
2856          * Due to this problem, we might experience lower throughput. The
2857          * suggested workaround is to disable DCTL[12:9] bits if we're
2858          * transitioning from U1/U2 to U0 and enable those bits again
2859          * after a transfer completes and there are no pending transfers
2860          * on any of the enabled endpoints.
2861          *
2862          * This is the first half of that workaround.
2863          *
2864          * Refers to:
2865          *
2866          * STAR#9000446952: RTL: Device SS : if U1/U2 ->U0 takes >128us
2867          * core send LGO_Ux entering U0
2868          */
2869         if (dwc->revision < DWC3_REVISION_183A) {
2870                 if (next == DWC3_LINK_STATE_U0) {
2871                         u32     u1u2;
2872                         u32     reg;
2873
2874                         switch (dwc->link_state) {
2875                         case DWC3_LINK_STATE_U1:
2876                         case DWC3_LINK_STATE_U2:
2877                                 reg = dwc3_readl(dwc->regs, DWC3_DCTL);
2878                                 u1u2 = reg & (DWC3_DCTL_INITU2ENA
2879                                                 | DWC3_DCTL_ACCEPTU2ENA
2880                                                 | DWC3_DCTL_INITU1ENA
2881                                                 | DWC3_DCTL_ACCEPTU1ENA);
2882
2883                                 if (!dwc->u1u2)
2884                                         dwc->u1u2 = reg & u1u2;
2885
2886                                 reg &= ~u1u2;
2887
2888                                 dwc3_writel(dwc->regs, DWC3_DCTL, reg);
2889                                 break;
2890                         default:
2891                                 /* do nothing */
2892                                 break;
2893                         }
2894                 }
2895         }
2896
2897         switch (next) {
2898         case DWC3_LINK_STATE_U1:
2899                 if (dwc->speed == USB_SPEED_SUPER)
2900                         dwc3_suspend_gadget(dwc);
2901                 break;
2902         case DWC3_LINK_STATE_U2:
2903         case DWC3_LINK_STATE_U3:
2904                 dwc3_suspend_gadget(dwc);
2905                 break;
2906         case DWC3_LINK_STATE_RESUME:
2907                 dwc3_resume_gadget(dwc);
2908                 break;
2909         default:
2910                 /* do nothing */
2911                 break;
2912         }
2913
2914         dwc->link_state = next;
2915 }
2916
2917 static void dwc3_gadget_suspend_interrupt(struct dwc3 *dwc,
2918                                           unsigned int evtinfo)
2919 {
2920         enum dwc3_link_state next = evtinfo & DWC3_LINK_STATE_MASK;
2921
2922         if (dwc->link_state != next && next == DWC3_LINK_STATE_U3)
2923                 dwc3_suspend_gadget(dwc);
2924
2925         dwc->link_state = next;
2926 }
2927
2928 static void dwc3_gadget_hibernation_interrupt(struct dwc3 *dwc,
2929                 unsigned int evtinfo)
2930 {
2931         unsigned int is_ss = evtinfo & BIT(4);
2932
2933         /*
2934          * WORKAROUND: DWC3 revison 2.20a with hibernation support
2935          * have a known issue which can cause USB CV TD.9.23 to fail
2936          * randomly.
2937          *
2938          * Because of this issue, core could generate bogus hibernation
2939          * events which SW needs to ignore.
2940          *
2941          * Refers to:
2942          *
2943          * STAR#9000546576: Device Mode Hibernation: Issue in USB 2.0
2944          * Device Fallback from SuperSpeed
2945          */
2946         if (is_ss ^ (dwc->speed == USB_SPEED_SUPER))
2947                 return;
2948
2949         /* enter hibernation here */
2950 }
2951
2952 static void dwc3_gadget_interrupt(struct dwc3 *dwc,
2953                 const struct dwc3_event_devt *event)
2954 {
2955         switch (event->type) {
2956         case DWC3_DEVICE_EVENT_DISCONNECT:
2957                 dwc3_gadget_disconnect_interrupt(dwc);
2958                 break;
2959         case DWC3_DEVICE_EVENT_RESET:
2960                 dwc3_gadget_reset_interrupt(dwc);
2961                 break;
2962         case DWC3_DEVICE_EVENT_CONNECT_DONE:
2963                 dwc3_gadget_conndone_interrupt(dwc);
2964                 break;
2965         case DWC3_DEVICE_EVENT_WAKEUP:
2966                 dwc3_gadget_wakeup_interrupt(dwc);
2967                 break;
2968         case DWC3_DEVICE_EVENT_HIBER_REQ:
2969                 if (dev_WARN_ONCE(dwc->dev, !dwc->has_hibernation,
2970                                         "unexpected hibernation event\n"))
2971                         break;
2972
2973                 dwc3_gadget_hibernation_interrupt(dwc, event->event_info);
2974                 break;
2975         case DWC3_DEVICE_EVENT_LINK_STATUS_CHANGE:
2976                 dwc3_gadget_linksts_change_interrupt(dwc, event->event_info);
2977                 break;
2978         case DWC3_DEVICE_EVENT_EOPF:
2979                 /* It changed to be suspend event for version 2.30a and above */
2980                 if (dwc->revision >= DWC3_REVISION_230A) {
2981                         /*
2982                          * Ignore suspend event until the gadget enters into
2983                          * USB_STATE_CONFIGURED state.
2984                          */
2985                         if (dwc->gadget.state >= USB_STATE_CONFIGURED)
2986                                 dwc3_gadget_suspend_interrupt(dwc,
2987                                                 event->event_info);
2988                 }
2989                 break;
2990         case DWC3_DEVICE_EVENT_SOF:
2991         case DWC3_DEVICE_EVENT_ERRATIC_ERROR:
2992         case DWC3_DEVICE_EVENT_CMD_CMPL:
2993         case DWC3_DEVICE_EVENT_OVERFLOW:
2994                 break;
2995         default:
2996                 dev_WARN(dwc->dev, "UNKNOWN IRQ %d\n", event->type);
2997         }
2998 }
2999
3000 static void dwc3_process_event_entry(struct dwc3 *dwc,
3001                 const union dwc3_event *event)
3002 {
3003         trace_dwc3_event(event->raw, dwc);
3004
3005         if (!event->type.is_devspec)
3006                 dwc3_endpoint_interrupt(dwc, &event->depevt);
3007         else if (event->type.type == DWC3_EVENT_TYPE_DEV)
3008                 dwc3_gadget_interrupt(dwc, &event->devt);
3009         else
3010                 dev_err(dwc->dev, "UNKNOWN IRQ type %d\n", event->raw);
3011 }
3012
3013 static irqreturn_t dwc3_process_event_buf(struct dwc3_event_buffer *evt)
3014 {
3015         struct dwc3 *dwc = evt->dwc;
3016         irqreturn_t ret = IRQ_NONE;
3017         int left;
3018         u32 reg;
3019
3020         left = evt->count;
3021
3022         if (!(evt->flags & DWC3_EVENT_PENDING))
3023                 return IRQ_NONE;
3024
3025         while (left > 0) {
3026                 union dwc3_event event;
3027
3028                 event.raw = *(u32 *) (evt->cache + evt->lpos);
3029
3030                 dwc3_process_event_entry(dwc, &event);
3031
3032                 /*
3033                  * FIXME we wrap around correctly to the next entry as
3034                  * almost all entries are 4 bytes in size. There is one
3035                  * entry which has 12 bytes which is a regular entry
3036                  * followed by 8 bytes data. ATM I don't know how
3037                  * things are organized if we get next to the a
3038                  * boundary so I worry about that once we try to handle
3039                  * that.
3040                  */
3041                 evt->lpos = (evt->lpos + 4) % evt->length;
3042                 left -= 4;
3043         }
3044
3045         evt->count = 0;
3046         evt->flags &= ~DWC3_EVENT_PENDING;
3047         ret = IRQ_HANDLED;
3048
3049         /* Unmask interrupt */
3050         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3051         reg &= ~DWC3_GEVNTSIZ_INTMASK;
3052         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3053
3054         if (dwc->imod_interval) {
3055                 dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), DWC3_GEVNTCOUNT_EHB);
3056                 dwc3_writel(dwc->regs, DWC3_DEV_IMOD(0), dwc->imod_interval);
3057         }
3058
3059         return ret;
3060 }
3061
3062 static irqreturn_t dwc3_thread_interrupt(int irq, void *_evt)
3063 {
3064         struct dwc3_event_buffer *evt = _evt;
3065         struct dwc3 *dwc = evt->dwc;
3066         unsigned long flags;
3067         irqreturn_t ret = IRQ_NONE;
3068
3069         spin_lock_irqsave(&dwc->lock, flags);
3070         ret = dwc3_process_event_buf(evt);
3071         spin_unlock_irqrestore(&dwc->lock, flags);
3072
3073         return ret;
3074 }
3075
3076 static irqreturn_t dwc3_check_event_buf(struct dwc3_event_buffer *evt)
3077 {
3078         struct dwc3 *dwc = evt->dwc;
3079         u32 amount;
3080         u32 count;
3081         u32 reg;
3082
3083         if (pm_runtime_suspended(dwc->dev)) {
3084                 pm_runtime_get(dwc->dev);
3085                 disable_irq_nosync(dwc->irq_gadget);
3086                 dwc->pending_events = true;
3087                 return IRQ_HANDLED;
3088         }
3089
3090         /*
3091          * With PCIe legacy interrupt, test shows that top-half irq handler can
3092          * be called again after HW interrupt deassertion. Check if bottom-half
3093          * irq event handler completes before caching new event to prevent
3094          * losing events.
3095          */
3096         if (evt->flags & DWC3_EVENT_PENDING)
3097                 return IRQ_HANDLED;
3098
3099         count = dwc3_readl(dwc->regs, DWC3_GEVNTCOUNT(0));
3100         count &= DWC3_GEVNTCOUNT_MASK;
3101         if (!count)
3102                 return IRQ_NONE;
3103
3104         evt->count = count;
3105         evt->flags |= DWC3_EVENT_PENDING;
3106
3107         /* Mask interrupt */
3108         reg = dwc3_readl(dwc->regs, DWC3_GEVNTSIZ(0));
3109         reg |= DWC3_GEVNTSIZ_INTMASK;
3110         dwc3_writel(dwc->regs, DWC3_GEVNTSIZ(0), reg);
3111
3112         amount = min(count, evt->length - evt->lpos);
3113         memcpy(evt->cache + evt->lpos, evt->buf + evt->lpos, amount);
3114
3115         if (amount < count)
3116                 memcpy(evt->cache, evt->buf, count - amount);
3117
3118         dwc3_writel(dwc->regs, DWC3_GEVNTCOUNT(0), count);
3119
3120         return IRQ_WAKE_THREAD;
3121 }
3122
3123 static irqreturn_t dwc3_interrupt(int irq, void *_evt)
3124 {
3125         struct dwc3_event_buffer        *evt = _evt;
3126
3127         return dwc3_check_event_buf(evt);
3128 }
3129
3130 static int dwc3_gadget_get_irq(struct dwc3 *dwc)
3131 {
3132         struct platform_device *dwc3_pdev = to_platform_device(dwc->dev);
3133         int irq;
3134
3135         irq = platform_get_irq_byname(dwc3_pdev, "peripheral");
3136         if (irq > 0)
3137                 goto out;
3138
3139         if (irq == -EPROBE_DEFER)
3140                 goto out;
3141
3142         irq = platform_get_irq_byname(dwc3_pdev, "dwc_usb3");
3143         if (irq > 0)
3144                 goto out;
3145
3146         if (irq == -EPROBE_DEFER)
3147                 goto out;
3148
3149         irq = platform_get_irq(dwc3_pdev, 0);
3150         if (irq > 0)
3151                 goto out;
3152
3153         if (irq != -EPROBE_DEFER)
3154                 dev_err(dwc->dev, "missing peripheral IRQ\n");
3155
3156         if (!irq)
3157                 irq = -EINVAL;
3158
3159 out:
3160         return irq;
3161 }
3162
3163 /**
3164  * dwc3_gadget_init - initializes gadget related registers
3165  * @dwc: pointer to our controller context structure
3166  *
3167  * Returns 0 on success otherwise negative errno.
3168  */
3169 int dwc3_gadget_init(struct dwc3 *dwc)
3170 {
3171         int ret;
3172         int irq;
3173
3174         irq = dwc3_gadget_get_irq(dwc);
3175         if (irq < 0) {
3176                 ret = irq;
3177                 goto err0;
3178         }
3179
3180         dwc->irq_gadget = irq;
3181
3182         dwc->ep0_trb = dma_alloc_coherent(dwc->sysdev,
3183                                           sizeof(*dwc->ep0_trb) * 2,
3184                                           &dwc->ep0_trb_addr, GFP_KERNEL);
3185         if (!dwc->ep0_trb) {
3186                 dev_err(dwc->dev, "failed to allocate ep0 trb\n");
3187                 ret = -ENOMEM;
3188                 goto err0;
3189         }
3190
3191         dwc->setup_buf = kzalloc(DWC3_EP0_SETUP_SIZE, GFP_KERNEL);
3192         if (!dwc->setup_buf) {
3193                 ret = -ENOMEM;
3194                 goto err1;
3195         }
3196
3197         dwc->bounce = dma_alloc_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE,
3198                         &dwc->bounce_addr, GFP_KERNEL);
3199         if (!dwc->bounce) {
3200                 ret = -ENOMEM;
3201                 goto err2;
3202         }
3203
3204         init_completion(&dwc->ep0_in_setup);
3205
3206         dwc->gadget.ops                 = &dwc3_gadget_ops;
3207         dwc->gadget.speed               = USB_SPEED_UNKNOWN;
3208         dwc->gadget.sg_supported        = true;
3209         dwc->gadget.name                = "dwc3-gadget";
3210         dwc->gadget.is_otg              = dwc->dr_mode == USB_DR_MODE_OTG;
3211
3212         /*
3213          * FIXME We might be setting max_speed to <SUPER, however versions
3214          * <2.20a of dwc3 have an issue with metastability (documented
3215          * elsewhere in this driver) which tells us we can't set max speed to
3216          * anything lower than SUPER.
3217          *
3218          * Because gadget.max_speed is only used by composite.c and function
3219          * drivers (i.e. it won't go into dwc3's registers) we are allowing this
3220          * to happen so we avoid sending SuperSpeed Capability descriptor
3221          * together with our BOS descriptor as that could confuse host into
3222          * thinking we can handle super speed.
3223          *
3224          * Note that, in fact, we won't even support GetBOS requests when speed
3225          * is less than super speed because we don't have means, yet, to tell
3226          * composite.c that we are USB 2.0 + LPM ECN.
3227          */
3228         if (dwc->revision < DWC3_REVISION_220A &&
3229             !dwc->dis_metastability_quirk)
3230                 dev_info(dwc->dev, "changing max_speed on rev %08x\n",
3231                                 dwc->revision);
3232
3233         dwc->gadget.max_speed           = dwc->maximum_speed;
3234
3235         /*
3236          * REVISIT: Here we should clear all pending IRQs to be
3237          * sure we're starting from a well known location.
3238          */
3239
3240         ret = dwc3_gadget_init_endpoints(dwc, dwc->num_eps);
3241         if (ret)
3242                 goto err3;
3243
3244         ret = usb_add_gadget_udc(dwc->dev, &dwc->gadget);
3245         if (ret) {
3246                 dev_err(dwc->dev, "failed to register udc\n");
3247                 goto err4;
3248         }
3249
3250         return 0;
3251
3252 err4:
3253         dwc3_gadget_free_endpoints(dwc);
3254
3255 err3:
3256         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3257                         dwc->bounce_addr);
3258
3259 err2:
3260         kfree(dwc->setup_buf);
3261
3262 err1:
3263         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3264                         dwc->ep0_trb, dwc->ep0_trb_addr);
3265
3266 err0:
3267         return ret;
3268 }
3269
3270 /* -------------------------------------------------------------------------- */
3271
3272 void dwc3_gadget_exit(struct dwc3 *dwc)
3273 {
3274         usb_del_gadget_udc(&dwc->gadget);
3275         dwc3_gadget_free_endpoints(dwc);
3276         dma_free_coherent(dwc->sysdev, DWC3_BOUNCE_SIZE, dwc->bounce,
3277                           dwc->bounce_addr);
3278         kfree(dwc->setup_buf);
3279         dma_free_coherent(dwc->sysdev, sizeof(*dwc->ep0_trb) * 2,
3280                           dwc->ep0_trb, dwc->ep0_trb_addr);
3281 }
3282
3283 int dwc3_gadget_suspend(struct dwc3 *dwc)
3284 {
3285         if (!dwc->gadget_driver)
3286                 return 0;
3287
3288         dwc3_gadget_run_stop(dwc, false, false);
3289         dwc3_disconnect_gadget(dwc);
3290         __dwc3_gadget_stop(dwc);
3291
3292         return 0;
3293 }
3294
3295 int dwc3_gadget_resume(struct dwc3 *dwc)
3296 {
3297         int                     ret;
3298
3299         if (!dwc->gadget_driver)
3300                 return 0;
3301
3302         ret = __dwc3_gadget_start(dwc);
3303         if (ret < 0)
3304                 goto err0;
3305
3306         ret = dwc3_gadget_run_stop(dwc, true, false);
3307         if (ret < 0)
3308                 goto err1;
3309
3310         return 0;
3311
3312 err1:
3313         __dwc3_gadget_stop(dwc);
3314
3315 err0:
3316         return ret;
3317 }
3318
3319 void dwc3_gadget_process_pending_events(struct dwc3 *dwc)
3320 {
3321         if (dwc->pending_events) {
3322                 dwc3_interrupt(dwc->irq_gadget, dwc->ev_buf);
3323                 dwc->pending_events = false;
3324                 enable_irq(dwc->irq_gadget);
3325         }
3326 }