drivers/net/ethernet/stmicro/stmmac/stmmac_main.c

   1 // SPDX-License-Identifier: GPL-2.0-only
   2 /*******************************************************************************
   3   This is the driver for the ST MAC 10/100/1000 on-chip Ethernet controllers.
   4   ST Ethernet IPs are built around a Synopsys IP Core.
   5
   6         Copyright(C) 2007-2011 STMicroelectronics Ltd
   7
   8
   9   Author: Giuseppe Cavallaro <peppe.cavallaro@st.com>
  10
  11   Documentation available at:
  12         http://www.stlinux.com
  13   Support available at:
  14         https://bugzilla.stlinux.com/
  15 *******************************************************************************/
  16
  17 #include <linux/clk.h>
  18 #include <linux/kernel.h>
  19 #include <linux/interrupt.h>
  20 #include <linux/ip.h>
  21 #include <linux/tcp.h>
  22 #include <linux/skbuff.h>
  23 #include <linux/ethtool.h>
  24 #include <linux/if_ether.h>
  25 #include <linux/crc32.h>
  26 #include <linux/mii.h>
  27 #include <linux/if.h>
  28 #include <linux/if_vlan.h>
  29 #include <linux/dma-mapping.h>
  30 #include <linux/slab.h>
  31 #include <linux/prefetch.h>
  32 #include <linux/pinctrl/consumer.h>
  33 #ifdef CONFIG_DEBUG_FS
  34 #include <linux/debugfs.h>
  35 #include <linux/seq_file.h>
  36 #endif /* CONFIG_DEBUG_FS */
  37 #include <linux/net_tstamp.h>
  38 #include <linux/phylink.h>
  39 #include <linux/udp.h>
  40 #include <net/pkt_cls.h>
  41 #include "stmmac_ptp.h"
  42 #include "stmmac.h"
  43 #include <linux/reset.h>
  44 #include <linux/of_mdio.h>
  45 #include "dwmac1000.h"
  46 #include "dwxgmac2.h"
  47 #include "hwif.h"
  48
  49 #define STMMAC_ALIGN(x)         __ALIGN_KERNEL(x, SMP_CACHE_BYTES)
  50 #define TSO_MAX_BUFF_SIZE       (SZ_16K - 1)
  51
  52 /* Module parameters */
  53 #define TX_TIMEO        5000
  54 static int watchdog = TX_TIMEO;
  55 module_param(watchdog, int, 0644);
  56 MODULE_PARM_DESC(watchdog, "Transmit timeout in milliseconds (default 5s)");
  57
  58 static int debug = -1;
  59 module_param(debug, int, 0644);
  60 MODULE_PARM_DESC(debug, "Message Level (-1: default, 0: no output, 16: all)");
  61
  62 static int phyaddr = -1;
  63 module_param(phyaddr, int, 0444);
  64 MODULE_PARM_DESC(phyaddr, "Physical device address");
  65
  66 #define STMMAC_TX_THRESH        (DMA_TX_SIZE / 4)
  67 #define STMMAC_RX_THRESH        (DMA_RX_SIZE / 4)
  68
  69 static int flow_ctrl = FLOW_AUTO;
  70 module_param(flow_ctrl, int, 0644);
  71 MODULE_PARM_DESC(flow_ctrl, "Flow control ability [on/off]");
  72
  73 static int pause = PAUSE_TIME;
  74 module_param(pause, int, 0644);
  75 MODULE_PARM_DESC(pause, "Flow Control Pause Time");
  76
  77 #define TC_DEFAULT 64
  78 static int tc = TC_DEFAULT;
  79 module_param(tc, int, 0644);
  80 MODULE_PARM_DESC(tc, "DMA threshold control value");
  81
  82 #define DEFAULT_BUFSIZE 1536
  83 static int buf_sz = DEFAULT_BUFSIZE;
  84 module_param(buf_sz, int, 0644);
  85 MODULE_PARM_DESC(buf_sz, "DMA buffer size");
  86
  87 #define STMMAC_RX_COPYBREAK     256
  88
  89 static const u32 default_msg_level = (NETIF_MSG_DRV | NETIF_MSG_PROBE |
  90                                       NETIF_MSG_LINK | NETIF_MSG_IFUP |
  91                                       NETIF_MSG_IFDOWN | NETIF_MSG_TIMER);
  92
  93 #define STMMAC_DEFAULT_LPI_TIMER        1000
  94 static int eee_timer = STMMAC_DEFAULT_LPI_TIMER;
  95 module_param(eee_timer, int, 0644);
  96 MODULE_PARM_DESC(eee_timer, "LPI tx expiration time in msec");
  97 #define STMMAC_LPI_T(x) (jiffies + msecs_to_jiffies(x))
  98
  99 /* By default the driver will use the ring mode to manage tx and rx descriptors,
 100  * but allow user to force to use the chain instead of the ring
 101  */
 102 static unsigned int chain_mode;
 103 module_param(chain_mode, int, 0444);
 104 MODULE_PARM_DESC(chain_mode, "To use chain instead of ring mode");
 105
 106 static irqreturn_t stmmac_interrupt(int irq, void *dev_id);
 107
 108 #ifdef CONFIG_DEBUG_FS
 109 static void stmmac_init_fs(struct net_device *dev);
 110 static void stmmac_exit_fs(struct net_device *dev);
 111 #endif
 112
 113 #define STMMAC_COAL_TIMER(x) (jiffies + usecs_to_jiffies(x))
 114
 115 /**
 116  * stmmac_verify_args - verify the driver parameters.
 117  * Description: it checks the driver parameters and set a default in case of
 118  * errors.
 119  */
 120 static void stmmac_verify_args(void)
 121 {
 122         if (unlikely(watchdog < 0))
 123                 watchdog = TX_TIMEO;
 124         if (unlikely((buf_sz < DEFAULT_BUFSIZE) || (buf_sz > BUF_SIZE_16KiB)))
 125                 buf_sz = DEFAULT_BUFSIZE;
 126         if (unlikely(flow_ctrl > 1))
 127                 flow_ctrl = FLOW_AUTO;
 128         else if (likely(flow_ctrl < 0))
 129                 flow_ctrl = FLOW_OFF;
 130         if (unlikely((pause < 0) || (pause > 0xffff)))
 131                 pause = PAUSE_TIME;
 132         if (eee_timer < 0)
 133                 eee_timer = STMMAC_DEFAULT_LPI_TIMER;
 134 }
 135
 136 /**
 137  * stmmac_disable_all_queues - Disable all queues
 138  * @priv: driver private structure
 139  */
 140 static void stmmac_disable_all_queues(struct stmmac_priv *priv)
 141 {
 142         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 143         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 144         u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
 145         u32 queue;
 146
 147         for (queue = 0; queue < maxq; queue++) {
 148                 struct stmmac_channel *ch = &priv->channel[queue];
 149
 150                 if (queue < rx_queues_cnt)
 151                         napi_disable(&ch->rx_napi);
 152                 if (queue < tx_queues_cnt)
 153                         napi_disable(&ch->tx_napi);
 154         }
 155 }
 156
 157 /**
 158  * stmmac_enable_all_queues - Enable all queues
 159  * @priv: driver private structure
 160  */
 161 static void stmmac_enable_all_queues(struct stmmac_priv *priv)
 162 {
 163         u32 rx_queues_cnt = priv->plat->rx_queues_to_use;
 164         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 165         u32 maxq = max(rx_queues_cnt, tx_queues_cnt);
 166         u32 queue;
 167
 168         for (queue = 0; queue < maxq; queue++) {
 169                 struct stmmac_channel *ch = &priv->channel[queue];
 170
 171                 if (queue < rx_queues_cnt)
 172                         napi_enable(&ch->rx_napi);
 173                 if (queue < tx_queues_cnt)
 174                         napi_enable(&ch->tx_napi);
 175         }
 176 }
 177
 178 /**
 179  * stmmac_stop_all_queues - Stop all queues
 180  * @priv: driver private structure
 181  */
 182 static void stmmac_stop_all_queues(struct stmmac_priv *priv)
 183 {
 184         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 185         u32 queue;
 186
 187         for (queue = 0; queue < tx_queues_cnt; queue++)
 188                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
 189 }
 190
 191 /**
 192  * stmmac_start_all_queues - Start all queues
 193  * @priv: driver private structure
 194  */
 195 static void stmmac_start_all_queues(struct stmmac_priv *priv)
 196 {
 197         u32 tx_queues_cnt = priv->plat->tx_queues_to_use;
 198         u32 queue;
 199
 200         for (queue = 0; queue < tx_queues_cnt; queue++)
 201                 netif_tx_start_queue(netdev_get_tx_queue(priv->dev, queue));
 202 }
 203
 204 static void stmmac_service_event_schedule(struct stmmac_priv *priv)
 205 {
 206         if (!test_bit(STMMAC_DOWN, &priv->state) &&
 207             !test_and_set_bit(STMMAC_SERVICE_SCHED, &priv->state))
 208                 queue_work(priv->wq, &priv->service_task);
 209 }
 210
 211 static void stmmac_global_err(struct stmmac_priv *priv)
 212 {
 213         netif_carrier_off(priv->dev);
 214         set_bit(STMMAC_RESET_REQUESTED, &priv->state);
 215         stmmac_service_event_schedule(priv);
 216 }
 217
 218 /**
 219  * stmmac_clk_csr_set - dynamically set the MDC clock
 220  * @priv: driver private structure
 221  * Description: this is to dynamically set the MDC clock according to the csr
 222  * clock input.
 223  * Note:
 224  *      If a specific clk_csr value is passed from the platform
 225  *      this means that the CSR Clock Range selection cannot be
 226  *      changed at run-time and it is fixed (as reported in the driver
 227  *      documentation). Viceversa the driver will try to set the MDC
 228  *      clock dynamically according to the actual clock input.
 229  */
 230 static void stmmac_clk_csr_set(struct stmmac_priv *priv)
 231 {
 232         u32 clk_rate;
 233
 234         clk_rate = clk_get_rate(priv->plat->stmmac_clk);
 235
 236         /* Platform provided default clk_csr would be assumed valid
 237          * for all other cases except for the below mentioned ones.
 238          * For values higher than the IEEE 802.3 specified frequency
 239          * we can not estimate the proper divider as it is not known
 240          * the frequency of clk_csr_i. So we do not change the default
 241          * divider.
 242          */
 243         if (!(priv->clk_csr & MAC_CSR_H_FRQ_MASK)) {
 244                 if (clk_rate < CSR_F_35M)
 245                         priv->clk_csr = STMMAC_CSR_20_35M;
 246                 else if ((clk_rate >= CSR_F_35M) && (clk_rate < CSR_F_60M))
 247                         priv->clk_csr = STMMAC_CSR_35_60M;
 248                 else if ((clk_rate >= CSR_F_60M) && (clk_rate < CSR_F_100M))
 249                         priv->clk_csr = STMMAC_CSR_60_100M;
 250                 else if ((clk_rate >= CSR_F_100M) && (clk_rate < CSR_F_150M))
 251                         priv->clk_csr = STMMAC_CSR_100_150M;
 252                 else if ((clk_rate >= CSR_F_150M) && (clk_rate < CSR_F_250M))
 253                         priv->clk_csr = STMMAC_CSR_150_250M;
 254                 else if ((clk_rate >= CSR_F_250M) && (clk_rate < CSR_F_300M))
 255                         priv->clk_csr = STMMAC_CSR_250_300M;
 256         }
 257
 258         if (priv->plat->has_sun8i) {
 259                 if (clk_rate > 160000000)
 260                         priv->clk_csr = 0x03;
 261                 else if (clk_rate > 80000000)
 262                         priv->clk_csr = 0x02;
 263                 else if (clk_rate > 40000000)
 264                         priv->clk_csr = 0x01;
 265                 else
 266                         priv->clk_csr = 0;
 267         }
 268
 269         if (priv->plat->has_xgmac) {
 270                 if (clk_rate > 400000000)
 271                         priv->clk_csr = 0x5;
 272                 else if (clk_rate > 350000000)
 273                         priv->clk_csr = 0x4;
 274                 else if (clk_rate > 300000000)
 275                         priv->clk_csr = 0x3;
 276                 else if (clk_rate > 250000000)
 277                         priv->clk_csr = 0x2;
 278                 else if (clk_rate > 150000000)
 279                         priv->clk_csr = 0x1;
 280                 else
 281                         priv->clk_csr = 0x0;
 282         }
 283 }
 284
 285 static void print_pkt(unsigned char *buf, int len)
 286 {
 287         pr_debug("len = %d byte, buf addr: 0x%p\n", len, buf);
 288         print_hex_dump_bytes("", DUMP_PREFIX_OFFSET, buf, len);
 289 }
 290
 291 static inline u32 stmmac_tx_avail(struct stmmac_priv *priv, u32 queue)
 292 {
 293         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 294         u32 avail;
 295
 296         if (tx_q->dirty_tx > tx_q->cur_tx)
 297                 avail = tx_q->dirty_tx - tx_q->cur_tx - 1;
 298         else
 299                 avail = DMA_TX_SIZE - tx_q->cur_tx + tx_q->dirty_tx - 1;
 300
 301         return avail;
 302 }
 303
 304 /**
 305  * stmmac_rx_dirty - Get RX queue dirty
 306  * @priv: driver private structure
 307  * @queue: RX queue index
 308  */
 309 static inline u32 stmmac_rx_dirty(struct stmmac_priv *priv, u32 queue)
 310 {
 311         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
 312         u32 dirty;
 313
 314         if (rx_q->dirty_rx <= rx_q->cur_rx)
 315                 dirty = rx_q->cur_rx - rx_q->dirty_rx;
 316         else
 317                 dirty = DMA_RX_SIZE - rx_q->dirty_rx + rx_q->cur_rx;
 318
 319         return dirty;
 320 }
 321
 322 /**
 323  * stmmac_enable_eee_mode - check and enter in LPI mode
 324  * @priv: driver private structure
 325  * Description: this function is to verify and enter in LPI mode in case of
 326  * EEE.
 327  */
 328 static void stmmac_enable_eee_mode(struct stmmac_priv *priv)
 329 {
 330         u32 tx_cnt = priv->plat->tx_queues_to_use;
 331         u32 queue;
 332
 333         /* check if all TX queues have the work finished */
 334         for (queue = 0; queue < tx_cnt; queue++) {
 335                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
 336
 337                 if (tx_q->dirty_tx != tx_q->cur_tx)
 338                         return; /* still unfinished work */
 339         }
 340
 341         /* Check and enter in LPI mode */
 342         if (!priv->tx_path_in_lpi_mode)
 343                 stmmac_set_eee_mode(priv, priv->hw,
 344                                 priv->plat->en_tx_lpi_clockgating);
 345 }
 346
 347 /**
 348  * stmmac_disable_eee_mode - disable and exit from LPI mode
 349  * @priv: driver private structure
 350  * Description: this function is to exit and disable EEE in case of
 351  * LPI state is true. This is called by the xmit.
 352  */
 353 void stmmac_disable_eee_mode(struct stmmac_priv *priv)
 354 {
 355         stmmac_reset_eee_mode(priv, priv->hw);
 356         del_timer_sync(&priv->eee_ctrl_timer);
 357         priv->tx_path_in_lpi_mode = false;
 358 }
 359
 360 /**
 361  * stmmac_eee_ctrl_timer - EEE TX SW timer.
 362  * @arg : data hook
 363  * Description:
 364  *  if there is no data transfer and if we are not in LPI state,
 365  *  then MAC Transmitter can be moved to LPI state.
 366  */
 367 static void stmmac_eee_ctrl_timer(struct timer_list *t)
 368 {
 369         struct stmmac_priv *priv = from_timer(priv, t, eee_ctrl_timer);
 370
 371         stmmac_enable_eee_mode(priv);
 372         mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 373 }
 374
 375 /**
 376  * stmmac_eee_init - init EEE
 377  * @priv: driver private structure
 378  * Description:
 379  *  if the GMAC supports the EEE (from the HW cap reg) and the phy device
 380  *  can also manage EEE, this function enable the LPI state and start related
 381  *  timer.
 382  */
 383 bool stmmac_eee_init(struct stmmac_priv *priv)
 384 {
 385         int tx_lpi_timer = priv->tx_lpi_timer;
 386
 387         /* Using PCS we cannot dial with the phy registers at this stage
 388          * so we do not support extra feature like EEE.
 389          */
 390         if ((priv->hw->pcs == STMMAC_PCS_RGMII) ||
 391             (priv->hw->pcs == STMMAC_PCS_TBI) ||
 392             (priv->hw->pcs == STMMAC_PCS_RTBI))
 393                 return false;
 394
 395         /* Check if MAC core supports the EEE feature. */
 396         if (!priv->dma_cap.eee)
 397                 return false;
 398
 399         mutex_lock(&priv->lock);
 400
 401         /* Check if it needs to be deactivated */
 402         if (!priv->eee_active) {
 403                 if (priv->eee_enabled) {
 404                         netdev_dbg(priv->dev, "disable EEE\n");
 405                         del_timer_sync(&priv->eee_ctrl_timer);
 406                         stmmac_set_eee_timer(priv, priv->hw, 0, tx_lpi_timer);
 407                 }
 408                 mutex_unlock(&priv->lock);
 409                 return false;
 410         }
 411
 412         if (priv->eee_active && !priv->eee_enabled) {
 413                 timer_setup(&priv->eee_ctrl_timer, stmmac_eee_ctrl_timer, 0);
 414                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
 415                 stmmac_set_eee_timer(priv, priv->hw, STMMAC_DEFAULT_LIT_LS,
 416                                      tx_lpi_timer);
 417         }
 418
 419         mutex_unlock(&priv->lock);
 420         netdev_dbg(priv->dev, "Energy-Efficient Ethernet initialized\n");
 421         return true;
 422 }
 423
 424 /* stmmac_get_tx_hwtstamp - get HW TX timestamps
 425  * @priv: driver private structure
 426  * @p : descriptor pointer
 427  * @skb : the socket buffer
 428  * Description :
 429  * This function will read timestamp from the descriptor & pass it to stack.
 430  * and also perform some sanity checks.
 431  */
 432 static void stmmac_get_tx_hwtstamp(struct stmmac_priv *priv,
 433                                    struct dma_desc *p, struct sk_buff *skb)
 434 {
 435         struct skb_shared_hwtstamps shhwtstamp;
 436         bool found = false;
 437         u64 ns = 0;
 438
 439         if (!priv->hwts_tx_en)
 440                 return;
 441
 442         /* exit if skb doesn't support hw tstamp */
 443         if (likely(!skb || !(skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS)))
 444                 return;
 445
 446         /* check tx tstamp status */
 447         if (stmmac_get_tx_timestamp_status(priv, p)) {
 448                 stmmac_get_timestamp(priv, p, priv->adv_ts, &ns);
 449                 found = true;
 450         } else if (!stmmac_get_mac_tx_timestamp(priv, priv->hw, &ns)) {
 451                 found = true;
 452         }
 453
 454         if (found) {
 455                 memset(&shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 456                 shhwtstamp.hwtstamp = ns_to_ktime(ns);
 457
 458                 netdev_dbg(priv->dev, "get valid TX hw timestamp %llu\n", ns);
 459                 /* pass tstamp to stack */
 460                 skb_tstamp_tx(skb, &shhwtstamp);
 461         }
 462 }
 463
 464 /* stmmac_get_rx_hwtstamp - get HW RX timestamps
 465  * @priv: driver private structure
 466  * @p : descriptor pointer
 467  * @np : next descriptor pointer
 468  * @skb : the socket buffer
 469  * Description :
 470  * This function will read received packet's timestamp from the descriptor
 471  * and pass it to stack. It also perform some sanity checks.
 472  */
 473 static void stmmac_get_rx_hwtstamp(struct stmmac_priv *priv, struct dma_desc *p,
 474                                    struct dma_desc *np, struct sk_buff *skb)
 475 {
 476         struct skb_shared_hwtstamps *shhwtstamp = NULL;
 477         struct dma_desc *desc = p;
 478         u64 ns = 0;
 479
 480         if (!priv->hwts_rx_en)
 481                 return;
 482         /* For GMAC4, the valid timestamp is from CTX next desc. */
 483         if (priv->plat->has_gmac4 || priv->plat->has_xgmac)
 484                 desc = np;
 485
 486         /* Check if timestamp is available */
 487         if (stmmac_get_rx_timestamp_status(priv, p, np, priv->adv_ts)) {
 488                 stmmac_get_timestamp(priv, desc, priv->adv_ts, &ns);
 489                 netdev_dbg(priv->dev, "get valid RX hw timestamp %llu\n", ns);
 490                 shhwtstamp = skb_hwtstamps(skb);
 491                 memset(shhwtstamp, 0, sizeof(struct skb_shared_hwtstamps));
 492                 shhwtstamp->hwtstamp = ns_to_ktime(ns);
 493         } else  {
 494                 netdev_dbg(priv->dev, "cannot get RX hw timestamp\n");
 495         }
 496 }
 497
 498 /**
 499  *  stmmac_hwtstamp_set - control hardware timestamping.
 500  *  @dev: device pointer.
 501  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 502  *  a proprietary structure used to pass information to the driver.
 503  *  Description:
 504  *  This function configures the MAC to enable/disable both outgoing(TX)
 505  *  and incoming(RX) packets time stamping based on user input.
 506  *  Return Value:
 507  *  0 on success and an appropriate -ve integer on failure.
 508  */
 509 static int stmmac_hwtstamp_set(struct net_device *dev, struct ifreq *ifr)
 510 {
 511         struct stmmac_priv *priv = netdev_priv(dev);
 512         struct hwtstamp_config config;
 513         struct timespec64 now;
 514         u64 temp = 0;
 515         u32 ptp_v2 = 0;
 516         u32 tstamp_all = 0;
 517         u32 ptp_over_ipv4_udp = 0;
 518         u32 ptp_over_ipv6_udp = 0;
 519         u32 ptp_over_ethernet = 0;
 520         u32 snap_type_sel = 0;
 521         u32 ts_master_en = 0;
 522         u32 ts_event_en = 0;
 523         u32 sec_inc = 0;
 524         u32 value = 0;
 525         bool xmac;
 526
 527         xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
 528
 529         if (!(priv->dma_cap.time_stamp || priv->adv_ts)) {
 530                 netdev_alert(priv->dev, "No support for HW time stamping\n");
 531                 priv->hwts_tx_en = 0;
 532                 priv->hwts_rx_en = 0;
 533
 534                 return -EOPNOTSUPP;
 535         }
 536
 537         if (copy_from_user(&config, ifr->ifr_data,
 538                            sizeof(config)))
 539                 return -EFAULT;
 540
 541         netdev_dbg(priv->dev, "%s config flags:0x%x, tx_type:0x%x, rx_filter:0x%x\n",
 542                    __func__, config.flags, config.tx_type, config.rx_filter);
 543
 544         /* reserved for future extensions */
 545         if (config.flags)
 546                 return -EINVAL;
 547
 548         if (config.tx_type != HWTSTAMP_TX_OFF &&
 549             config.tx_type != HWTSTAMP_TX_ON)
 550                 return -ERANGE;
 551
 552         if (priv->adv_ts) {
 553                 switch (config.rx_filter) {
 554                 case HWTSTAMP_FILTER_NONE:
 555                         /* time stamp no incoming packet at all */
 556                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 557                         break;
 558
 559                 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT:
 560                         /* PTP v1, UDP, any kind of event packet */
 561                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 562                         /* 'xmac' hardware can support Sync, Pdelay_Req and
 563                          * Pdelay_resp by setting bit14 and bits17/16 to 01
 564                          * This leaves Delay_Req timestamps out.
 565                          * Enable all events *and* general purpose message
 566                          * timestamping
 567                          */
 568                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 569                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 570                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 571                         break;
 572
 573                 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC:
 574                         /* PTP v1, UDP, Sync packet */
 575                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_SYNC;
 576                         /* take time stamp for SYNC messages only */
 577                         ts_event_en = PTP_TCR_TSEVNTENA;
 578
 579                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 580                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 581                         break;
 582
 583                 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ:
 584                         /* PTP v1, UDP, Delay_req packet */
 585                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ;
 586                         /* take time stamp for Delay_Req messages only */
 587                         ts_master_en = PTP_TCR_TSMSTRENA;
 588                         ts_event_en = PTP_TCR_TSEVNTENA;
 589
 590                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 591                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 592                         break;
 593
 594                 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT:
 595                         /* PTP v2, UDP, any kind of event packet */
 596                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_EVENT;
 597                         ptp_v2 = PTP_TCR_TSVER2ENA;
 598                         /* take time stamp for all event messages */
 599                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 600
 601                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 602                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 603                         break;
 604
 605                 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC:
 606                         /* PTP v2, UDP, Sync packet */
 607                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_SYNC;
 608                         ptp_v2 = PTP_TCR_TSVER2ENA;
 609                         /* take time stamp for SYNC messages only */
 610                         ts_event_en = PTP_TCR_TSEVNTENA;
 611
 612                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 613                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 614                         break;
 615
 616                 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ:
 617                         /* PTP v2, UDP, Delay_req packet */
 618                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ;
 619                         ptp_v2 = PTP_TCR_TSVER2ENA;
 620                         /* take time stamp for Delay_Req messages only */
 621                         ts_master_en = PTP_TCR_TSMSTRENA;
 622                         ts_event_en = PTP_TCR_TSEVNTENA;
 623
 624                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 625                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 626                         break;
 627
 628                 case HWTSTAMP_FILTER_PTP_V2_EVENT:
 629                         /* PTP v2/802.AS1 any layer, any kind of event packet */
 630                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_EVENT;
 631                         ptp_v2 = PTP_TCR_TSVER2ENA;
 632                         snap_type_sel = PTP_TCR_SNAPTYPSEL_1;
 633                         ts_event_en = PTP_TCR_TSEVNTENA;
 634                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 635                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 636                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 637                         break;
 638
 639                 case HWTSTAMP_FILTER_PTP_V2_SYNC:
 640                         /* PTP v2/802.AS1, any layer, Sync packet */
 641                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_SYNC;
 642                         ptp_v2 = PTP_TCR_TSVER2ENA;
 643                         /* take time stamp for SYNC messages only */
 644                         ts_event_en = PTP_TCR_TSEVNTENA;
 645
 646                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 647                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 648                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 649                         break;
 650
 651                 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ:
 652                         /* PTP v2/802.AS1, any layer, Delay_req packet */
 653                         config.rx_filter = HWTSTAMP_FILTER_PTP_V2_DELAY_REQ;
 654                         ptp_v2 = PTP_TCR_TSVER2ENA;
 655                         /* take time stamp for Delay_Req messages only */
 656                         ts_master_en = PTP_TCR_TSMSTRENA;
 657                         ts_event_en = PTP_TCR_TSEVNTENA;
 658
 659                         ptp_over_ipv4_udp = PTP_TCR_TSIPV4ENA;
 660                         ptp_over_ipv6_udp = PTP_TCR_TSIPV6ENA;
 661                         ptp_over_ethernet = PTP_TCR_TSIPENA;
 662                         break;
 663
 664                 case HWTSTAMP_FILTER_NTP_ALL:
 665                 case HWTSTAMP_FILTER_ALL:
 666                         /* time stamp any incoming packet */
 667                         config.rx_filter = HWTSTAMP_FILTER_ALL;
 668                         tstamp_all = PTP_TCR_TSENALL;
 669                         break;
 670
 671                 default:
 672                         return -ERANGE;
 673                 }
 674         } else {
 675                 switch (config.rx_filter) {
 676                 case HWTSTAMP_FILTER_NONE:
 677                         config.rx_filter = HWTSTAMP_FILTER_NONE;
 678                         break;
 679                 default:
 680                         /* PTP v1, UDP, any kind of event packet */
 681                         config.rx_filter = HWTSTAMP_FILTER_PTP_V1_L4_EVENT;
 682                         break;
 683                 }
 684         }
 685         priv->hwts_rx_en = ((config.rx_filter == HWTSTAMP_FILTER_NONE) ? 0 : 1);
 686         priv->hwts_tx_en = config.tx_type == HWTSTAMP_TX_ON;
 687
 688         if (!priv->hwts_tx_en && !priv->hwts_rx_en)
 689                 stmmac_config_hw_tstamping(priv, priv->ptpaddr, 0);
 690         else {
 691                 value = (PTP_TCR_TSENA | PTP_TCR_TSCFUPDT | PTP_TCR_TSCTRLSSR |
 692                          tstamp_all | ptp_v2 | ptp_over_ethernet |
 693                          ptp_over_ipv6_udp | ptp_over_ipv4_udp | ts_event_en |
 694                          ts_master_en | snap_type_sel);
 695                 stmmac_config_hw_tstamping(priv, priv->ptpaddr, value);
 696
 697                 /* program Sub Second Increment reg */
 698                 stmmac_config_sub_second_increment(priv,
 699                                 priv->ptpaddr, priv->plat->clk_ptp_rate,
 700                                 xmac, &sec_inc);
 701                 temp = div_u64(1000000000ULL, sec_inc);
 702
 703                 /* Store sub second increment and flags for later use */
 704                 priv->sub_second_inc = sec_inc;
 705                 priv->systime_flags = value;
 706
 707                 /* calculate default added value:
 708                  * formula is :
 709                  * addend = (2^32)/freq_div_ratio;
 710                  * where, freq_div_ratio = 1e9ns/sec_inc
 711                  */
 712                 temp = (u64)(temp << 32);
 713                 priv->default_addend = div_u64(temp, priv->plat->clk_ptp_rate);
 714                 stmmac_config_addend(priv, priv->ptpaddr, priv->default_addend);
 715
 716                 /* initialize system time */
 717                 ktime_get_real_ts64(&now);
 718
 719                 /* lower 32 bits of tv_sec are safe until y2106 */
 720                 stmmac_init_systime(priv, priv->ptpaddr,
 721                                 (u32)now.tv_sec, now.tv_nsec);
 722         }
 723
 724         memcpy(&priv->tstamp_config, &config, sizeof(config));
 725
 726         return copy_to_user(ifr->ifr_data, &config,
 727                             sizeof(config)) ? -EFAULT : 0;
 728 }
 729
 730 /**
 731  *  stmmac_hwtstamp_get - read hardware timestamping.
 732  *  @dev: device pointer.
 733  *  @ifr: An IOCTL specific structure, that can contain a pointer to
 734  *  a proprietary structure used to pass information to the driver.
 735  *  Description:
 736  *  This function obtain the current hardware timestamping settings
 737     as requested.
 738  */
 739 static int stmmac_hwtstamp_get(struct net_device *dev, struct ifreq *ifr)
 740 {
 741         struct stmmac_priv *priv = netdev_priv(dev);
 742         struct hwtstamp_config *config = &priv->tstamp_config;
 743
 744         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 745                 return -EOPNOTSUPP;
 746
 747         return copy_to_user(ifr->ifr_data, config,
 748                             sizeof(*config)) ? -EFAULT : 0;
 749 }
 750
 751 /**
 752  * stmmac_init_ptp - init PTP
 753  * @priv: driver private structure
 754  * Description: this is to verify if the HW supports the PTPv1 or PTPv2.
 755  * This is done by looking at the HW cap. register.
 756  * This function also registers the ptp driver.
 757  */
 758 static int stmmac_init_ptp(struct stmmac_priv *priv)
 759 {
 760         bool xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
 761
 762         if (!(priv->dma_cap.time_stamp || priv->dma_cap.atime_stamp))
 763                 return -EOPNOTSUPP;
 764
 765         priv->adv_ts = 0;
 766         /* Check if adv_ts can be enabled for dwmac 4.x / xgmac core */
 767         if (xmac && priv->dma_cap.atime_stamp)
 768                 priv->adv_ts = 1;
 769         /* Dwmac 3.x core with extend_desc can support adv_ts */
 770         else if (priv->extend_desc && priv->dma_cap.atime_stamp)
 771                 priv->adv_ts = 1;
 772
 773         if (priv->dma_cap.time_stamp)
 774                 netdev_info(priv->dev, "IEEE 1588-2002 Timestamp supported\n");
 775
 776         if (priv->adv_ts)
 777                 netdev_info(priv->dev,
 778                             "IEEE 1588-2008 Advanced Timestamp supported\n");
 779
 780         priv->hwts_tx_en = 0;
 781         priv->hwts_rx_en = 0;
 782
 783         stmmac_ptp_register(priv);
 784
 785         return 0;
 786 }
 787
 788 static void stmmac_release_ptp(struct stmmac_priv *priv)
 789 {
 790         if (priv->plat->clk_ptp_ref)
 791                 clk_disable_unprepare(priv->plat->clk_ptp_ref);
 792         stmmac_ptp_unregister(priv);
 793 }
 794
 795 /**
 796  *  stmmac_mac_flow_ctrl - Configure flow control in all queues
 797  *  @priv: driver private structure
 798  *  Description: It is used for configuring the flow control in all queues
 799  */
 800 static void stmmac_mac_flow_ctrl(struct stmmac_priv *priv, u32 duplex)
 801 {
 802         u32 tx_cnt = priv->plat->tx_queues_to_use;
 803
 804         stmmac_flow_ctrl(priv, priv->hw, duplex, priv->flow_ctrl,
 805                         priv->pause, tx_cnt);
 806 }
 807
 808 static void stmmac_validate(struct phylink_config *config,
 809                             unsigned long *supported,
 810                             struct phylink_link_state *state)
 811 {
 812         struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
 813         __ETHTOOL_DECLARE_LINK_MODE_MASK(mac_supported) = { 0, };
 814         __ETHTOOL_DECLARE_LINK_MODE_MASK(mask) = { 0, };
 815         int tx_cnt = priv->plat->tx_queues_to_use;
 816         int max_speed = priv->plat->max_speed;
 817
 818         phylink_set(mac_supported, 10baseT_Half);
 819         phylink_set(mac_supported, 10baseT_Full);
 820         phylink_set(mac_supported, 100baseT_Half);
 821         phylink_set(mac_supported, 100baseT_Full);
 822         phylink_set(mac_supported, 1000baseT_Half);
 823         phylink_set(mac_supported, 1000baseT_Full);
 824         phylink_set(mac_supported, 1000baseKX_Full);
 825
 826         phylink_set(mac_supported, Autoneg);
 827         phylink_set(mac_supported, Pause);
 828         phylink_set(mac_supported, Asym_Pause);
 829         phylink_set_port_modes(mac_supported);
 830
 831         /* Cut down 1G if asked to */
 832         if ((max_speed > 0) && (max_speed < 1000)) {
 833                 phylink_set(mask, 1000baseT_Full);
 834                 phylink_set(mask, 1000baseX_Full);
 835         } else if (priv->plat->has_xgmac) {
 836                 if (!max_speed || (max_speed >= 2500)) {
 837                         phylink_set(mac_supported, 2500baseT_Full);
 838                         phylink_set(mac_supported, 2500baseX_Full);
 839                 }
 840                 if (!max_speed || (max_speed >= 5000)) {
 841                         phylink_set(mac_supported, 5000baseT_Full);
 842                 }
 843                 if (!max_speed || (max_speed >= 10000)) {
 844                         phylink_set(mac_supported, 10000baseSR_Full);
 845                         phylink_set(mac_supported, 10000baseLR_Full);
 846                         phylink_set(mac_supported, 10000baseER_Full);
 847                         phylink_set(mac_supported, 10000baseLRM_Full);
 848                         phylink_set(mac_supported, 10000baseT_Full);
 849                         phylink_set(mac_supported, 10000baseKX4_Full);
 850                         phylink_set(mac_supported, 10000baseKR_Full);
 851                 }
 852         }
 853
 854         /* Half-Duplex can only work with single queue */
 855         if (tx_cnt > 1) {
 856                 phylink_set(mask, 10baseT_Half);
 857                 phylink_set(mask, 100baseT_Half);
 858                 phylink_set(mask, 1000baseT_Half);
 859         }
 860
 861         bitmap_and(supported, supported, mac_supported,
 862                    __ETHTOOL_LINK_MODE_MASK_NBITS);
 863         bitmap_andnot(supported, supported, mask,
 864                       __ETHTOOL_LINK_MODE_MASK_NBITS);
 865         bitmap_and(state->advertising, state->advertising, mac_supported,
 866                    __ETHTOOL_LINK_MODE_MASK_NBITS);
 867         bitmap_andnot(state->advertising, state->advertising, mask,
 868                       __ETHTOOL_LINK_MODE_MASK_NBITS);
 869 }
 870
 871 static int stmmac_mac_link_state(struct phylink_config *config,
 872                                  struct phylink_link_state *state)
 873 {
 874         return -EOPNOTSUPP;
 875 }
 876
 877 static void stmmac_mac_config(struct phylink_config *config, unsigned int mode,
 878                               const struct phylink_link_state *state)
 879 {
 880         struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
 881         u32 ctrl;
 882
 883         ctrl = readl(priv->ioaddr + MAC_CTRL_REG);
 884         ctrl &= ~priv->hw->link.speed_mask;
 885
 886         if (state->interface == PHY_INTERFACE_MODE_USXGMII) {
 887                 switch (state->speed) {
 888                 case SPEED_10000:
 889                         ctrl |= priv->hw->link.xgmii.speed10000;
 890                         break;
 891                 case SPEED_5000:
 892                         ctrl |= priv->hw->link.xgmii.speed5000;
 893                         break;
 894                 case SPEED_2500:
 895                         ctrl |= priv->hw->link.xgmii.speed2500;
 896                         break;
 897                 default:
 898                         return;
 899                 }
 900         } else {
 901                 switch (state->speed) {
 902                 case SPEED_2500:
 903                         ctrl |= priv->hw->link.speed2500;
 904                         break;
 905                 case SPEED_1000:
 906                         ctrl |= priv->hw->link.speed1000;
 907                         break;
 908                 case SPEED_100:
 909                         ctrl |= priv->hw->link.speed100;
 910                         break;
 911                 case SPEED_10:
 912                         ctrl |= priv->hw->link.speed10;
 913                         break;
 914                 default:
 915                         return;
 916                 }
 917         }
 918
 919         priv->speed = state->speed;
 920
 921         if (priv->plat->fix_mac_speed)
 922                 priv->plat->fix_mac_speed(priv->plat->bsp_priv, state->speed);
 923
 924         if (!state->duplex)
 925                 ctrl &= ~priv->hw->link.duplex;
 926         else
 927                 ctrl |= priv->hw->link.duplex;
 928
 929         /* Flow Control operation */
 930         if (state->pause)
 931                 stmmac_mac_flow_ctrl(priv, state->duplex);
 932
 933         writel(ctrl, priv->ioaddr + MAC_CTRL_REG);
 934 }
 935
 936 static void stmmac_mac_an_restart(struct phylink_config *config)
 937 {
 938         /* Not Supported */
 939 }
 940
 941 static void stmmac_mac_link_down(struct phylink_config *config,
 942                                  unsigned int mode, phy_interface_t interface)
 943 {
 944         struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
 945
 946         stmmac_mac_set(priv, priv->ioaddr, false);
 947         priv->eee_active = false;
 948         stmmac_eee_init(priv);
 949         stmmac_set_eee_pls(priv, priv->hw, false);
 950 }
 951
 952 static void stmmac_mac_link_up(struct phylink_config *config,
 953                                unsigned int mode, phy_interface_t interface,
 954                                struct phy_device *phy)
 955 {
 956         struct stmmac_priv *priv = netdev_priv(to_net_dev(config->dev));
 957
 958         stmmac_mac_set(priv, priv->ioaddr, true);
 959         if (phy && priv->dma_cap.eee) {
 960                 priv->eee_active = phy_init_eee(phy, 1) >= 0;
 961                 priv->eee_enabled = stmmac_eee_init(priv);
 962                 stmmac_set_eee_pls(priv, priv->hw, true);
 963         }
 964 }
 965
 966 static const struct phylink_mac_ops stmmac_phylink_mac_ops = {
 967         .validate = stmmac_validate,
 968         .mac_link_state = stmmac_mac_link_state,
 969         .mac_config = stmmac_mac_config,
 970         .mac_an_restart = stmmac_mac_an_restart,
 971         .mac_link_down = stmmac_mac_link_down,
 972         .mac_link_up = stmmac_mac_link_up,
 973 };
 974
 975 /**
 976  * stmmac_check_pcs_mode - verify if RGMII/SGMII is supported
 977  * @priv: driver private structure
 978  * Description: this is to verify if the HW supports the PCS.
 979  * Physical Coding Sublayer (PCS) interface that can be used when the MAC is
 980  * configured for the TBI, RTBI, or SGMII PHY interface.
 981  */
 982 static void stmmac_check_pcs_mode(struct stmmac_priv *priv)
 983 {
 984         int interface = priv->plat->interface;
 985
 986         if (priv->dma_cap.pcs) {
 987                 if ((interface == PHY_INTERFACE_MODE_RGMII) ||
 988                     (interface == PHY_INTERFACE_MODE_RGMII_ID) ||
 989                     (interface == PHY_INTERFACE_MODE_RGMII_RXID) ||
 990                     (interface == PHY_INTERFACE_MODE_RGMII_TXID)) {
 991                         netdev_dbg(priv->dev, "PCS RGMII support enabled\n");
 992                         priv->hw->pcs = STMMAC_PCS_RGMII;
 993                 } else if (interface == PHY_INTERFACE_MODE_SGMII) {
 994                         netdev_dbg(priv->dev, "PCS SGMII support enabled\n");
 995                         priv->hw->pcs = STMMAC_PCS_SGMII;
 996                 }
 997         }
 998 }
 999
1000 /**
1001  * stmmac_init_phy - PHY initialization
1002  * @dev: net device structure
1003  * Description: it initializes the driver's PHY state, and attaches the PHY
1004  * to the mac driver.
1005  *  Return value:
1006  *  0 on success
1007  */
1008 static int stmmac_init_phy(struct net_device *dev)
1009 {
1010         struct stmmac_priv *priv = netdev_priv(dev);
1011         struct device_node *node;
1012         int ret;
1013
1014         node = priv->plat->phylink_node;
1015
1016         if (node)
1017                 ret = phylink_of_phy_connect(priv->phylink, node, 0);
1018
1019         /* Some DT bindings do not set-up the PHY handle. Let's try to
1020          * manually parse it
1021          */
1022         if (!node || ret) {
1023                 int addr = priv->plat->phy_addr;
1024                 struct phy_device *phydev;
1025
1026                 phydev = mdiobus_get_phy(priv->mii, addr);
1027                 if (!phydev) {
1028                         netdev_err(priv->dev, "no phy at addr %d\n", addr);
1029                         return -ENODEV;
1030                 }
1031
1032                 ret = phylink_connect_phy(priv->phylink, phydev);
1033         }
1034
1035         return ret;
1036 }
1037
1038 static int stmmac_phy_setup(struct stmmac_priv *priv)
1039 {
1040         struct fwnode_handle *fwnode = of_fwnode_handle(priv->plat->phylink_node);
1041         int mode = priv->plat->phy_interface;
1042         struct phylink *phylink;
1043
1044         priv->phylink_config.dev = &priv->dev->dev;
1045         priv->phylink_config.type = PHYLINK_NETDEV;
1046
1047         phylink = phylink_create(&priv->phylink_config, fwnode,
1048                                  mode, &stmmac_phylink_mac_ops);
1049         if (IS_ERR(phylink))
1050                 return PTR_ERR(phylink);
1051
1052         priv->phylink = phylink;
1053         return 0;
1054 }
1055
1056 static void stmmac_display_rx_rings(struct stmmac_priv *priv)
1057 {
1058         u32 rx_cnt = priv->plat->rx_queues_to_use;
1059         void *head_rx;
1060         u32 queue;
1061
1062         /* Display RX rings */
1063         for (queue = 0; queue < rx_cnt; queue++) {
1064                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1065
1066                 pr_info("\tRX Queue %u rings\n", queue);
1067
1068                 if (priv->extend_desc)
1069                         head_rx = (void *)rx_q->dma_erx;
1070                 else
1071                         head_rx = (void *)rx_q->dma_rx;
1072
1073                 /* Display RX ring */
1074                 stmmac_display_ring(priv, head_rx, DMA_RX_SIZE, true);
1075         }
1076 }
1077
1078 static void stmmac_display_tx_rings(struct stmmac_priv *priv)
1079 {
1080         u32 tx_cnt = priv->plat->tx_queues_to_use;
1081         void *head_tx;
1082         u32 queue;
1083
1084         /* Display TX rings */
1085         for (queue = 0; queue < tx_cnt; queue++) {
1086                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1087
1088                 pr_info("\tTX Queue %d rings\n", queue);
1089
1090                 if (priv->extend_desc)
1091                         head_tx = (void *)tx_q->dma_etx;
1092                 else
1093                         head_tx = (void *)tx_q->dma_tx;
1094
1095                 stmmac_display_ring(priv, head_tx, DMA_TX_SIZE, false);
1096         }
1097 }
1098
1099 static void stmmac_display_rings(struct stmmac_priv *priv)
1100 {
1101         /* Display RX ring */
1102         stmmac_display_rx_rings(priv);
1103
1104         /* Display TX ring */
1105         stmmac_display_tx_rings(priv);
1106 }
1107
1108 static int stmmac_set_bfsize(int mtu, int bufsize)
1109 {
1110         int ret = bufsize;
1111
1112         if (mtu >= BUF_SIZE_4KiB)
1113                 ret = BUF_SIZE_8KiB;
1114         else if (mtu >= BUF_SIZE_2KiB)
1115                 ret = BUF_SIZE_4KiB;
1116         else if (mtu > DEFAULT_BUFSIZE)
1117                 ret = BUF_SIZE_2KiB;
1118         else
1119                 ret = DEFAULT_BUFSIZE;
1120
1121         return ret;
1122 }
1123
1124 /**
1125  * stmmac_clear_rx_descriptors - clear RX descriptors
1126  * @priv: driver private structure
1127  * @queue: RX queue index
1128  * Description: this function is called to clear the RX descriptors
1129  * in case of both basic and extended descriptors are used.
1130  */
1131 static void stmmac_clear_rx_descriptors(struct stmmac_priv *priv, u32 queue)
1132 {
1133         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1134         int i;
1135
1136         /* Clear the RX descriptors */
1137         for (i = 0; i < DMA_RX_SIZE; i++)
1138                 if (priv->extend_desc)
1139                         stmmac_init_rx_desc(priv, &rx_q->dma_erx[i].basic,
1140                                         priv->use_riwt, priv->mode,
1141                                         (i == DMA_RX_SIZE - 1),
1142                                         priv->dma_buf_sz);
1143                 else
1144                         stmmac_init_rx_desc(priv, &rx_q->dma_rx[i],
1145                                         priv->use_riwt, priv->mode,
1146                                         (i == DMA_RX_SIZE - 1),
1147                                         priv->dma_buf_sz);
1148 }
1149
1150 /**
1151  * stmmac_clear_tx_descriptors - clear tx descriptors
1152  * @priv: driver private structure
1153  * @queue: TX queue index.
1154  * Description: this function is called to clear the TX descriptors
1155  * in case of both basic and extended descriptors are used.
1156  */
1157 static void stmmac_clear_tx_descriptors(struct stmmac_priv *priv, u32 queue)
1158 {
1159         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1160         int i;
1161
1162         /* Clear the TX descriptors */
1163         for (i = 0; i < DMA_TX_SIZE; i++)
1164                 if (priv->extend_desc)
1165                         stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
1166                                         priv->mode, (i == DMA_TX_SIZE - 1));
1167                 else
1168                         stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
1169                                         priv->mode, (i == DMA_TX_SIZE - 1));
1170 }
1171
1172 /**
1173  * stmmac_clear_descriptors - clear descriptors
1174  * @priv: driver private structure
1175  * Description: this function is called to clear the TX and RX descriptors
1176  * in case of both basic and extended descriptors are used.
1177  */
1178 static void stmmac_clear_descriptors(struct stmmac_priv *priv)
1179 {
1180         u32 rx_queue_cnt = priv->plat->rx_queues_to_use;
1181         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1182         u32 queue;
1183
1184         /* Clear the RX descriptors */
1185         for (queue = 0; queue < rx_queue_cnt; queue++)
1186                 stmmac_clear_rx_descriptors(priv, queue);
1187
1188         /* Clear the TX descriptors */
1189         for (queue = 0; queue < tx_queue_cnt; queue++)
1190                 stmmac_clear_tx_descriptors(priv, queue);
1191 }
1192
1193 /**
1194  * stmmac_init_rx_buffers - init the RX descriptor buffer.
1195  * @priv: driver private structure
1196  * @p: descriptor pointer
1197  * @i: descriptor index
1198  * @flags: gfp flag
1199  * @queue: RX queue index
1200  * Description: this function is called to allocate a receive buffer, perform
1201  * the DMA mapping and init the descriptor.
1202  */
1203 static int stmmac_init_rx_buffers(struct stmmac_priv *priv, struct dma_desc *p,
1204                                   int i, gfp_t flags, u32 queue)
1205 {
1206         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1207         struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1208
1209         buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
1210         if (!buf->page)
1211                 return -ENOMEM;
1212
1213         if (priv->sph) {
1214                 buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
1215                 if (!buf->sec_page)
1216                         return -ENOMEM;
1217
1218                 buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
1219                 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
1220         } else {
1221                 buf->sec_page = NULL;
1222         }
1223
1224         buf->addr = page_pool_get_dma_addr(buf->page);
1225         stmmac_set_desc_addr(priv, p, buf->addr);
1226         if (priv->dma_buf_sz == BUF_SIZE_16KiB)
1227                 stmmac_init_desc3(priv, p);
1228
1229         return 0;
1230 }
1231
1232 /**
1233  * stmmac_free_rx_buffer - free RX dma buffers
1234  * @priv: private structure
1235  * @queue: RX queue index
1236  * @i: buffer index.
1237  */
1238 static void stmmac_free_rx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1239 {
1240         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1241         struct stmmac_rx_buffer *buf = &rx_q->buf_pool[i];
1242
1243         if (buf->page)
1244                 page_pool_put_page(rx_q->page_pool, buf->page, false);
1245         buf->page = NULL;
1246
1247         if (buf->sec_page)
1248                 page_pool_put_page(rx_q->page_pool, buf->sec_page, false);
1249         buf->sec_page = NULL;
1250 }
1251
1252 /**
1253  * stmmac_free_tx_buffer - free RX dma buffers
1254  * @priv: private structure
1255  * @queue: RX queue index
1256  * @i: buffer index.
1257  */
1258 static void stmmac_free_tx_buffer(struct stmmac_priv *priv, u32 queue, int i)
1259 {
1260         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1261
1262         if (tx_q->tx_skbuff_dma[i].buf) {
1263                 if (tx_q->tx_skbuff_dma[i].map_as_page)
1264                         dma_unmap_page(priv->device,
1265                                        tx_q->tx_skbuff_dma[i].buf,
1266                                        tx_q->tx_skbuff_dma[i].len,
1267                                        DMA_TO_DEVICE);
1268                 else
1269                         dma_unmap_single(priv->device,
1270                                          tx_q->tx_skbuff_dma[i].buf,
1271                                          tx_q->tx_skbuff_dma[i].len,
1272                                          DMA_TO_DEVICE);
1273         }
1274
1275         if (tx_q->tx_skbuff[i]) {
1276                 dev_kfree_skb_any(tx_q->tx_skbuff[i]);
1277                 tx_q->tx_skbuff[i] = NULL;
1278                 tx_q->tx_skbuff_dma[i].buf = 0;
1279                 tx_q->tx_skbuff_dma[i].map_as_page = false;
1280         }
1281 }
1282
1283 /**
1284  * init_dma_rx_desc_rings - init the RX descriptor rings
1285  * @dev: net device structure
1286  * @flags: gfp flag.
1287  * Description: this function initializes the DMA RX descriptors
1288  * and allocates the socket buffers. It supports the chained and ring
1289  * modes.
1290  */
1291 static int init_dma_rx_desc_rings(struct net_device *dev, gfp_t flags)
1292 {
1293         struct stmmac_priv *priv = netdev_priv(dev);
1294         u32 rx_count = priv->plat->rx_queues_to_use;
1295         int ret = -ENOMEM;
1296         int bfsize = 0;
1297         int queue;
1298         int i;
1299
1300         bfsize = stmmac_set_16kib_bfsize(priv, dev->mtu);
1301         if (bfsize < 0)
1302                 bfsize = 0;
1303
1304         if (bfsize < BUF_SIZE_16KiB)
1305                 bfsize = stmmac_set_bfsize(dev->mtu, priv->dma_buf_sz);
1306
1307         priv->dma_buf_sz = bfsize;
1308
1309         /* RX INITIALIZATION */
1310         netif_dbg(priv, probe, priv->dev,
1311                   "SKB addresses:\nskb\t\tskb data\tdma data\n");
1312
1313         for (queue = 0; queue < rx_count; queue++) {
1314                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1315
1316                 netif_dbg(priv, probe, priv->dev,
1317                           "(%s) dma_rx_phy=0x%08x\n", __func__,
1318                           (u32)rx_q->dma_rx_phy);
1319
1320                 stmmac_clear_rx_descriptors(priv, queue);
1321
1322                 for (i = 0; i < DMA_RX_SIZE; i++) {
1323                         struct dma_desc *p;
1324
1325                         if (priv->extend_desc)
1326                                 p = &((rx_q->dma_erx + i)->basic);
1327                         else
1328                                 p = rx_q->dma_rx + i;
1329
1330                         ret = stmmac_init_rx_buffers(priv, p, i, flags,
1331                                                      queue);
1332                         if (ret)
1333                                 goto err_init_rx_buffers;
1334                 }
1335
1336                 rx_q->cur_rx = 0;
1337                 rx_q->dirty_rx = (unsigned int)(i - DMA_RX_SIZE);
1338
1339                 /* Setup the chained descriptor addresses */
1340                 if (priv->mode == STMMAC_CHAIN_MODE) {
1341                         if (priv->extend_desc)
1342                                 stmmac_mode_init(priv, rx_q->dma_erx,
1343                                                 rx_q->dma_rx_phy, DMA_RX_SIZE, 1);
1344                         else
1345                                 stmmac_mode_init(priv, rx_q->dma_rx,
1346                                                 rx_q->dma_rx_phy, DMA_RX_SIZE, 0);
1347                 }
1348         }
1349
1350         buf_sz = bfsize;
1351
1352         return 0;
1353
1354 err_init_rx_buffers:
1355         while (queue >= 0) {
1356                 while (--i >= 0)
1357                         stmmac_free_rx_buffer(priv, queue, i);
1358
1359                 if (queue == 0)
1360                         break;
1361
1362                 i = DMA_RX_SIZE;
1363                 queue--;
1364         }
1365
1366         return ret;
1367 }
1368
1369 /**
1370  * init_dma_tx_desc_rings - init the TX descriptor rings
1371  * @dev: net device structure.
1372  * Description: this function initializes the DMA TX descriptors
1373  * and allocates the socket buffers. It supports the chained and ring
1374  * modes.
1375  */
1376 static int init_dma_tx_desc_rings(struct net_device *dev)
1377 {
1378         struct stmmac_priv *priv = netdev_priv(dev);
1379         u32 tx_queue_cnt = priv->plat->tx_queues_to_use;
1380         u32 queue;
1381         int i;
1382
1383         for (queue = 0; queue < tx_queue_cnt; queue++) {
1384                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1385
1386                 netif_dbg(priv, probe, priv->dev,
1387                           "(%s) dma_tx_phy=0x%08x\n", __func__,
1388                          (u32)tx_q->dma_tx_phy);
1389
1390                 /* Setup the chained descriptor addresses */
1391                 if (priv->mode == STMMAC_CHAIN_MODE) {
1392                         if (priv->extend_desc)
1393                                 stmmac_mode_init(priv, tx_q->dma_etx,
1394                                                 tx_q->dma_tx_phy, DMA_TX_SIZE, 1);
1395                         else
1396                                 stmmac_mode_init(priv, tx_q->dma_tx,
1397                                                 tx_q->dma_tx_phy, DMA_TX_SIZE, 0);
1398                 }
1399
1400                 for (i = 0; i < DMA_TX_SIZE; i++) {
1401                         struct dma_desc *p;
1402                         if (priv->extend_desc)
1403                                 p = &((tx_q->dma_etx + i)->basic);
1404                         else
1405                                 p = tx_q->dma_tx + i;
1406
1407                         stmmac_clear_desc(priv, p);
1408
1409                         tx_q->tx_skbuff_dma[i].buf = 0;
1410                         tx_q->tx_skbuff_dma[i].map_as_page = false;
1411                         tx_q->tx_skbuff_dma[i].len = 0;
1412                         tx_q->tx_skbuff_dma[i].last_segment = false;
1413                         tx_q->tx_skbuff[i] = NULL;
1414                 }
1415
1416                 tx_q->dirty_tx = 0;
1417                 tx_q->cur_tx = 0;
1418                 tx_q->mss = 0;
1419
1420                 netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, queue));
1421         }
1422
1423         return 0;
1424 }
1425
1426 /**
1427  * init_dma_desc_rings - init the RX/TX descriptor rings
1428  * @dev: net device structure
1429  * @flags: gfp flag.
1430  * Description: this function initializes the DMA RX/TX descriptors
1431  * and allocates the socket buffers. It supports the chained and ring
1432  * modes.
1433  */
1434 static int init_dma_desc_rings(struct net_device *dev, gfp_t flags)
1435 {
1436         struct stmmac_priv *priv = netdev_priv(dev);
1437         int ret;
1438
1439         ret = init_dma_rx_desc_rings(dev, flags);
1440         if (ret)
1441                 return ret;
1442
1443         ret = init_dma_tx_desc_rings(dev);
1444
1445         stmmac_clear_descriptors(priv);
1446
1447         if (netif_msg_hw(priv))
1448                 stmmac_display_rings(priv);
1449
1450         return ret;
1451 }
1452
1453 /**
1454  * dma_free_rx_skbufs - free RX dma buffers
1455  * @priv: private structure
1456  * @queue: RX queue index
1457  */
1458 static void dma_free_rx_skbufs(struct stmmac_priv *priv, u32 queue)
1459 {
1460         int i;
1461
1462         for (i = 0; i < DMA_RX_SIZE; i++)
1463                 stmmac_free_rx_buffer(priv, queue, i);
1464 }
1465
1466 /**
1467  * dma_free_tx_skbufs - free TX dma buffers
1468  * @priv: private structure
1469  * @queue: TX queue index
1470  */
1471 static void dma_free_tx_skbufs(struct stmmac_priv *priv, u32 queue)
1472 {
1473         int i;
1474
1475         for (i = 0; i < DMA_TX_SIZE; i++)
1476                 stmmac_free_tx_buffer(priv, queue, i);
1477 }
1478
1479 /**
1480  * free_dma_rx_desc_resources - free RX dma desc resources
1481  * @priv: private structure
1482  */
1483 static void free_dma_rx_desc_resources(struct stmmac_priv *priv)
1484 {
1485         u32 rx_count = priv->plat->rx_queues_to_use;
1486         u32 queue;
1487
1488         /* Free RX queue resources */
1489         for (queue = 0; queue < rx_count; queue++) {
1490                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1491
1492                 /* Release the DMA RX socket buffers */
1493                 dma_free_rx_skbufs(priv, queue);
1494
1495                 /* Free DMA regions of consistent memory previously allocated */
1496                 if (!priv->extend_desc)
1497                         dma_free_coherent(priv->device,
1498                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1499                                           rx_q->dma_rx, rx_q->dma_rx_phy);
1500                 else
1501                         dma_free_coherent(priv->device, DMA_RX_SIZE *
1502                                           sizeof(struct dma_extended_desc),
1503                                           rx_q->dma_erx, rx_q->dma_rx_phy);
1504
1505                 kfree(rx_q->buf_pool);
1506                 if (rx_q->page_pool) {
1507                         page_pool_request_shutdown(rx_q->page_pool);
1508                         page_pool_destroy(rx_q->page_pool);
1509                 }
1510         }
1511 }
1512
1513 /**
1514  * free_dma_tx_desc_resources - free TX dma desc resources
1515  * @priv: private structure
1516  */
1517 static void free_dma_tx_desc_resources(struct stmmac_priv *priv)
1518 {
1519         u32 tx_count = priv->plat->tx_queues_to_use;
1520         u32 queue;
1521
1522         /* Free TX queue resources */
1523         for (queue = 0; queue < tx_count; queue++) {
1524                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1525
1526                 /* Release the DMA TX socket buffers */
1527                 dma_free_tx_skbufs(priv, queue);
1528
1529                 /* Free DMA regions of consistent memory previously allocated */
1530                 if (!priv->extend_desc)
1531                         dma_free_coherent(priv->device,
1532                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1533                                           tx_q->dma_tx, tx_q->dma_tx_phy);
1534                 else
1535                         dma_free_coherent(priv->device, DMA_TX_SIZE *
1536                                           sizeof(struct dma_extended_desc),
1537                                           tx_q->dma_etx, tx_q->dma_tx_phy);
1538
1539                 kfree(tx_q->tx_skbuff_dma);
1540                 kfree(tx_q->tx_skbuff);
1541         }
1542 }
1543
1544 /**
1545  * alloc_dma_rx_desc_resources - alloc RX resources.
1546  * @priv: private structure
1547  * Description: according to which descriptor can be used (extend or basic)
1548  * this function allocates the resources for TX and RX paths. In case of
1549  * reception, for example, it pre-allocated the RX socket buffer in order to
1550  * allow zero-copy mechanism.
1551  */
1552 static int alloc_dma_rx_desc_resources(struct stmmac_priv *priv)
1553 {
1554         u32 rx_count = priv->plat->rx_queues_to_use;
1555         int ret = -ENOMEM;
1556         u32 queue;
1557
1558         /* RX queues buffers and DMA */
1559         for (queue = 0; queue < rx_count; queue++) {
1560                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
1561                 struct page_pool_params pp_params = { 0 };
1562                 unsigned int num_pages;
1563
1564                 rx_q->queue_index = queue;
1565                 rx_q->priv_data = priv;
1566
1567                 pp_params.flags = PP_FLAG_DMA_MAP;
1568                 pp_params.pool_size = DMA_RX_SIZE;
1569                 num_pages = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE);
1570                 pp_params.order = ilog2(num_pages);
1571                 pp_params.nid = dev_to_node(priv->device);
1572                 pp_params.dev = priv->device;
1573                 pp_params.dma_dir = DMA_FROM_DEVICE;
1574
1575                 rx_q->page_pool = page_pool_create(&pp_params);
1576                 if (IS_ERR(rx_q->page_pool)) {
1577                         ret = PTR_ERR(rx_q->page_pool);
1578                         rx_q->page_pool = NULL;
1579                         goto err_dma;
1580                 }
1581
1582                 rx_q->buf_pool = kcalloc(DMA_RX_SIZE, sizeof(*rx_q->buf_pool),
1583                                          GFP_KERNEL);
1584                 if (!rx_q->buf_pool)
1585                         goto err_dma;
1586
1587                 if (priv->extend_desc) {
1588                         rx_q->dma_erx = dma_alloc_coherent(priv->device,
1589                                                            DMA_RX_SIZE * sizeof(struct dma_extended_desc),
1590                                                            &rx_q->dma_rx_phy,
1591                                                            GFP_KERNEL);
1592                         if (!rx_q->dma_erx)
1593                                 goto err_dma;
1594
1595                 } else {
1596                         rx_q->dma_rx = dma_alloc_coherent(priv->device,
1597                                                           DMA_RX_SIZE * sizeof(struct dma_desc),
1598                                                           &rx_q->dma_rx_phy,
1599                                                           GFP_KERNEL);
1600                         if (!rx_q->dma_rx)
1601                                 goto err_dma;
1602                 }
1603         }
1604
1605         return 0;
1606
1607 err_dma:
1608         free_dma_rx_desc_resources(priv);
1609
1610         return ret;
1611 }
1612
1613 /**
1614  * alloc_dma_tx_desc_resources - alloc TX resources.
1615  * @priv: private structure
1616  * Description: according to which descriptor can be used (extend or basic)
1617  * this function allocates the resources for TX and RX paths. In case of
1618  * reception, for example, it pre-allocated the RX socket buffer in order to
1619  * allow zero-copy mechanism.
1620  */
1621 static int alloc_dma_tx_desc_resources(struct stmmac_priv *priv)
1622 {
1623         u32 tx_count = priv->plat->tx_queues_to_use;
1624         int ret = -ENOMEM;
1625         u32 queue;
1626
1627         /* TX queues buffers and DMA */
1628         for (queue = 0; queue < tx_count; queue++) {
1629                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1630
1631                 tx_q->queue_index = queue;
1632                 tx_q->priv_data = priv;
1633
1634                 tx_q->tx_skbuff_dma = kcalloc(DMA_TX_SIZE,
1635                                               sizeof(*tx_q->tx_skbuff_dma),
1636                                               GFP_KERNEL);
1637                 if (!tx_q->tx_skbuff_dma)
1638                         goto err_dma;
1639
1640                 tx_q->tx_skbuff = kcalloc(DMA_TX_SIZE,
1641                                           sizeof(struct sk_buff *),
1642                                           GFP_KERNEL);
1643                 if (!tx_q->tx_skbuff)
1644                         goto err_dma;
1645
1646                 if (priv->extend_desc) {
1647                         tx_q->dma_etx = dma_alloc_coherent(priv->device,
1648                                                            DMA_TX_SIZE * sizeof(struct dma_extended_desc),
1649                                                            &tx_q->dma_tx_phy,
1650                                                            GFP_KERNEL);
1651                         if (!tx_q->dma_etx)
1652                                 goto err_dma;
1653                 } else {
1654                         tx_q->dma_tx = dma_alloc_coherent(priv->device,
1655                                                           DMA_TX_SIZE * sizeof(struct dma_desc),
1656                                                           &tx_q->dma_tx_phy,
1657                                                           GFP_KERNEL);
1658                         if (!tx_q->dma_tx)
1659                                 goto err_dma;
1660                 }
1661         }
1662
1663         return 0;
1664
1665 err_dma:
1666         free_dma_tx_desc_resources(priv);
1667
1668         return ret;
1669 }
1670
1671 /**
1672  * alloc_dma_desc_resources - alloc TX/RX resources.
1673  * @priv: private structure
1674  * Description: according to which descriptor can be used (extend or basic)
1675  * this function allocates the resources for TX and RX paths. In case of
1676  * reception, for example, it pre-allocated the RX socket buffer in order to
1677  * allow zero-copy mechanism.
1678  */
1679 static int alloc_dma_desc_resources(struct stmmac_priv *priv)
1680 {
1681         /* RX Allocation */
1682         int ret = alloc_dma_rx_desc_resources(priv);
1683
1684         if (ret)
1685                 return ret;
1686
1687         ret = alloc_dma_tx_desc_resources(priv);
1688
1689         return ret;
1690 }
1691
1692 /**
1693  * free_dma_desc_resources - free dma desc resources
1694  * @priv: private structure
1695  */
1696 static void free_dma_desc_resources(struct stmmac_priv *priv)
1697 {
1698         /* Release the DMA RX socket buffers */
1699         free_dma_rx_desc_resources(priv);
1700
1701         /* Release the DMA TX socket buffers */
1702         free_dma_tx_desc_resources(priv);
1703 }
1704
1705 /**
1706  *  stmmac_mac_enable_rx_queues - Enable MAC rx queues
1707  *  @priv: driver private structure
1708  *  Description: It is used for enabling the rx queues in the MAC
1709  */
1710 static void stmmac_mac_enable_rx_queues(struct stmmac_priv *priv)
1711 {
1712         u32 rx_queues_count = priv->plat->rx_queues_to_use;
1713         int queue;
1714         u8 mode;
1715
1716         for (queue = 0; queue < rx_queues_count; queue++) {
1717                 mode = priv->plat->rx_queues_cfg[queue].mode_to_use;
1718                 stmmac_rx_queue_enable(priv, priv->hw, mode, queue);
1719         }
1720 }
1721
1722 /**
1723  * stmmac_start_rx_dma - start RX DMA channel
1724  * @priv: driver private structure
1725  * @chan: RX channel index
1726  * Description:
1727  * This starts a RX DMA channel
1728  */
1729 static void stmmac_start_rx_dma(struct stmmac_priv *priv, u32 chan)
1730 {
1731         netdev_dbg(priv->dev, "DMA RX processes started in channel %d\n", chan);
1732         stmmac_start_rx(priv, priv->ioaddr, chan);
1733 }
1734
1735 /**
1736  * stmmac_start_tx_dma - start TX DMA channel
1737  * @priv: driver private structure
1738  * @chan: TX channel index
1739  * Description:
1740  * This starts a TX DMA channel
1741  */
1742 static void stmmac_start_tx_dma(struct stmmac_priv *priv, u32 chan)
1743 {
1744         netdev_dbg(priv->dev, "DMA TX processes started in channel %d\n", chan);
1745         stmmac_start_tx(priv, priv->ioaddr, chan);
1746 }
1747
1748 /**
1749  * stmmac_stop_rx_dma - stop RX DMA channel
1750  * @priv: driver private structure
1751  * @chan: RX channel index
1752  * Description:
1753  * This stops a RX DMA channel
1754  */
1755 static void stmmac_stop_rx_dma(struct stmmac_priv *priv, u32 chan)
1756 {
1757         netdev_dbg(priv->dev, "DMA RX processes stopped in channel %d\n", chan);
1758         stmmac_stop_rx(priv, priv->ioaddr, chan);
1759 }
1760
1761 /**
1762  * stmmac_stop_tx_dma - stop TX DMA channel
1763  * @priv: driver private structure
1764  * @chan: TX channel index
1765  * Description:
1766  * This stops a TX DMA channel
1767  */
1768 static void stmmac_stop_tx_dma(struct stmmac_priv *priv, u32 chan)
1769 {
1770         netdev_dbg(priv->dev, "DMA TX processes stopped in channel %d\n", chan);
1771         stmmac_stop_tx(priv, priv->ioaddr, chan);
1772 }
1773
1774 /**
1775  * stmmac_start_all_dma - start all RX and TX DMA channels
1776  * @priv: driver private structure
1777  * Description:
1778  * This starts all the RX and TX DMA channels
1779  */
1780 static void stmmac_start_all_dma(struct stmmac_priv *priv)
1781 {
1782         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1783         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1784         u32 chan = 0;
1785
1786         for (chan = 0; chan < rx_channels_count; chan++)
1787                 stmmac_start_rx_dma(priv, chan);
1788
1789         for (chan = 0; chan < tx_channels_count; chan++)
1790                 stmmac_start_tx_dma(priv, chan);
1791 }
1792
1793 /**
1794  * stmmac_stop_all_dma - stop all RX and TX DMA channels
1795  * @priv: driver private structure
1796  * Description:
1797  * This stops the RX and TX DMA channels
1798  */
1799 static void stmmac_stop_all_dma(struct stmmac_priv *priv)
1800 {
1801         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1802         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1803         u32 chan = 0;
1804
1805         for (chan = 0; chan < rx_channels_count; chan++)
1806                 stmmac_stop_rx_dma(priv, chan);
1807
1808         for (chan = 0; chan < tx_channels_count; chan++)
1809                 stmmac_stop_tx_dma(priv, chan);
1810 }
1811
1812 /**
1813  *  stmmac_dma_operation_mode - HW DMA operation mode
1814  *  @priv: driver private structure
1815  *  Description: it is used for configuring the DMA operation mode register in
1816  *  order to program the tx/rx DMA thresholds or Store-And-Forward mode.
1817  */
1818 static void stmmac_dma_operation_mode(struct stmmac_priv *priv)
1819 {
1820         u32 rx_channels_count = priv->plat->rx_queues_to_use;
1821         u32 tx_channels_count = priv->plat->tx_queues_to_use;
1822         int rxfifosz = priv->plat->rx_fifo_size;
1823         int txfifosz = priv->plat->tx_fifo_size;
1824         u32 txmode = 0;
1825         u32 rxmode = 0;
1826         u32 chan = 0;
1827         u8 qmode = 0;
1828
1829         if (rxfifosz == 0)
1830                 rxfifosz = priv->dma_cap.rx_fifo_size;
1831         if (txfifosz == 0)
1832                 txfifosz = priv->dma_cap.tx_fifo_size;
1833
1834         /* Adjust for real per queue fifo size */
1835         rxfifosz /= rx_channels_count;
1836         txfifosz /= tx_channels_count;
1837
1838         if (priv->plat->force_thresh_dma_mode) {
1839                 txmode = tc;
1840                 rxmode = tc;
1841         } else if (priv->plat->force_sf_dma_mode || priv->plat->tx_coe) {
1842                 /*
1843                  * In case of GMAC, SF mode can be enabled
1844                  * to perform the TX COE in HW. This depends on:
1845                  * 1) TX COE if actually supported
1846                  * 2) There is no bugged Jumbo frame support
1847                  *    that needs to not insert csum in the TDES.
1848                  */
1849                 txmode = SF_DMA_MODE;
1850                 rxmode = SF_DMA_MODE;
1851                 priv->xstats.threshold = SF_DMA_MODE;
1852         } else {
1853                 txmode = tc;
1854                 rxmode = SF_DMA_MODE;
1855         }
1856
1857         /* configure all channels */
1858         for (chan = 0; chan < rx_channels_count; chan++) {
1859                 qmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
1860
1861                 stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan,
1862                                 rxfifosz, qmode);
1863                 stmmac_set_dma_bfsize(priv, priv->ioaddr, priv->dma_buf_sz,
1864                                 chan);
1865         }
1866
1867         for (chan = 0; chan < tx_channels_count; chan++) {
1868                 qmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
1869
1870                 stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan,
1871                                 txfifosz, qmode);
1872         }
1873 }
1874
1875 /**
1876  * stmmac_tx_clean - to manage the transmission completion
1877  * @priv: driver private structure
1878  * @queue: TX queue index
1879  * Description: it reclaims the transmit resources after transmission completes.
1880  */
1881 static int stmmac_tx_clean(struct stmmac_priv *priv, int budget, u32 queue)
1882 {
1883         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
1884         unsigned int bytes_compl = 0, pkts_compl = 0;
1885         unsigned int entry, count = 0;
1886
1887         __netif_tx_lock_bh(netdev_get_tx_queue(priv->dev, queue));
1888
1889         priv->xstats.tx_clean++;
1890
1891         entry = tx_q->dirty_tx;
1892         while ((entry != tx_q->cur_tx) && (count < budget)) {
1893                 struct sk_buff *skb = tx_q->tx_skbuff[entry];
1894                 struct dma_desc *p;
1895                 int status;
1896
1897                 if (priv->extend_desc)
1898                         p = (struct dma_desc *)(tx_q->dma_etx + entry);
1899                 else
1900                         p = tx_q->dma_tx + entry;
1901
1902                 status = stmmac_tx_status(priv, &priv->dev->stats,
1903                                 &priv->xstats, p, priv->ioaddr);
1904                 /* Check if the descriptor is owned by the DMA */
1905                 if (unlikely(status & tx_dma_own))
1906                         break;
1907
1908                 count++;
1909
1910                 /* Make sure descriptor fields are read after reading
1911                  * the own bit.
1912                  */
1913                 dma_rmb();
1914
1915                 /* Just consider the last segment and ...*/
1916                 if (likely(!(status & tx_not_ls))) {
1917                         /* ... verify the status error condition */
1918                         if (unlikely(status & tx_err)) {
1919                                 priv->dev->stats.tx_errors++;
1920                         } else {
1921                                 priv->dev->stats.tx_packets++;
1922                                 priv->xstats.tx_pkt_n++;
1923                         }
1924                         stmmac_get_tx_hwtstamp(priv, p, skb);
1925                 }
1926
1927                 if (likely(tx_q->tx_skbuff_dma[entry].buf)) {
1928                         if (tx_q->tx_skbuff_dma[entry].map_as_page)
1929                                 dma_unmap_page(priv->device,
1930                                                tx_q->tx_skbuff_dma[entry].buf,
1931                                                tx_q->tx_skbuff_dma[entry].len,
1932                                                DMA_TO_DEVICE);
1933                         else
1934                                 dma_unmap_single(priv->device,
1935                                                  tx_q->tx_skbuff_dma[entry].buf,
1936                                                  tx_q->tx_skbuff_dma[entry].len,
1937                                                  DMA_TO_DEVICE);
1938                         tx_q->tx_skbuff_dma[entry].buf = 0;
1939                         tx_q->tx_skbuff_dma[entry].len = 0;
1940                         tx_q->tx_skbuff_dma[entry].map_as_page = false;
1941                 }
1942
1943                 stmmac_clean_desc3(priv, tx_q, p);
1944
1945                 tx_q->tx_skbuff_dma[entry].last_segment = false;
1946                 tx_q->tx_skbuff_dma[entry].is_jumbo = false;
1947
1948                 if (likely(skb != NULL)) {
1949                         pkts_compl++;
1950                         bytes_compl += skb->len;
1951                         dev_consume_skb_any(skb);
1952                         tx_q->tx_skbuff[entry] = NULL;
1953                 }
1954
1955                 stmmac_release_tx_desc(priv, p, priv->mode);
1956
1957                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
1958         }
1959         tx_q->dirty_tx = entry;
1960
1961         netdev_tx_completed_queue(netdev_get_tx_queue(priv->dev, queue),
1962                                   pkts_compl, bytes_compl);
1963
1964         if (unlikely(netif_tx_queue_stopped(netdev_get_tx_queue(priv->dev,
1965                                                                 queue))) &&
1966             stmmac_tx_avail(priv, queue) > STMMAC_TX_THRESH) {
1967
1968                 netif_dbg(priv, tx_done, priv->dev,
1969                           "%s: restart transmit\n", __func__);
1970                 netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, queue));
1971         }
1972
1973         if ((priv->eee_enabled) && (!priv->tx_path_in_lpi_mode)) {
1974                 stmmac_enable_eee_mode(priv);
1975                 mod_timer(&priv->eee_ctrl_timer, STMMAC_LPI_T(eee_timer));
1976         }
1977
1978         /* We still have pending packets, let's call for a new scheduling */
1979         if (tx_q->dirty_tx != tx_q->cur_tx)
1980                 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(10));
1981
1982         __netif_tx_unlock_bh(netdev_get_tx_queue(priv->dev, queue));
1983
1984         return count;
1985 }
1986
1987 /**
1988  * stmmac_tx_err - to manage the tx error
1989  * @priv: driver private structure
1990  * @chan: channel index
1991  * Description: it cleans the descriptors and restarts the transmission
1992  * in case of transmission errors.
1993  */
1994 static void stmmac_tx_err(struct stmmac_priv *priv, u32 chan)
1995 {
1996         struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
1997         int i;
1998
1999         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, chan));
2000
2001         stmmac_stop_tx_dma(priv, chan);
2002         dma_free_tx_skbufs(priv, chan);
2003         for (i = 0; i < DMA_TX_SIZE; i++)
2004                 if (priv->extend_desc)
2005                         stmmac_init_tx_desc(priv, &tx_q->dma_etx[i].basic,
2006                                         priv->mode, (i == DMA_TX_SIZE - 1));
2007                 else
2008                         stmmac_init_tx_desc(priv, &tx_q->dma_tx[i],
2009                                         priv->mode, (i == DMA_TX_SIZE - 1));
2010         tx_q->dirty_tx = 0;
2011         tx_q->cur_tx = 0;
2012         tx_q->mss = 0;
2013         netdev_tx_reset_queue(netdev_get_tx_queue(priv->dev, chan));
2014         stmmac_start_tx_dma(priv, chan);
2015
2016         priv->dev->stats.tx_errors++;
2017         netif_tx_wake_queue(netdev_get_tx_queue(priv->dev, chan));
2018 }
2019
2020 /**
2021  *  stmmac_set_dma_operation_mode - Set DMA operation mode by channel
2022  *  @priv: driver private structure
2023  *  @txmode: TX operating mode
2024  *  @rxmode: RX operating mode
2025  *  @chan: channel index
2026  *  Description: it is used for configuring of the DMA operation mode in
2027  *  runtime in order to program the tx/rx DMA thresholds or Store-And-Forward
2028  *  mode.
2029  */
2030 static void stmmac_set_dma_operation_mode(struct stmmac_priv *priv, u32 txmode,
2031                                           u32 rxmode, u32 chan)
2032 {
2033         u8 rxqmode = priv->plat->rx_queues_cfg[chan].mode_to_use;
2034         u8 txqmode = priv->plat->tx_queues_cfg[chan].mode_to_use;
2035         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2036         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2037         int rxfifosz = priv->plat->rx_fifo_size;
2038         int txfifosz = priv->plat->tx_fifo_size;
2039
2040         if (rxfifosz == 0)
2041                 rxfifosz = priv->dma_cap.rx_fifo_size;
2042         if (txfifosz == 0)
2043                 txfifosz = priv->dma_cap.tx_fifo_size;
2044
2045         /* Adjust for real per queue fifo size */
2046         rxfifosz /= rx_channels_count;
2047         txfifosz /= tx_channels_count;
2048
2049         stmmac_dma_rx_mode(priv, priv->ioaddr, rxmode, chan, rxfifosz, rxqmode);
2050         stmmac_dma_tx_mode(priv, priv->ioaddr, txmode, chan, txfifosz, txqmode);
2051 }
2052
2053 static bool stmmac_safety_feat_interrupt(struct stmmac_priv *priv)
2054 {
2055         int ret;
2056
2057         ret = stmmac_safety_feat_irq_status(priv, priv->dev,
2058                         priv->ioaddr, priv->dma_cap.asp, &priv->sstats);
2059         if (ret && (ret != -EINVAL)) {
2060                 stmmac_global_err(priv);
2061                 return true;
2062         }
2063
2064         return false;
2065 }
2066
2067 static int stmmac_napi_check(struct stmmac_priv *priv, u32 chan)
2068 {
2069         int status = stmmac_dma_interrupt_status(priv, priv->ioaddr,
2070                                                  &priv->xstats, chan);
2071         struct stmmac_channel *ch = &priv->channel[chan];
2072
2073         if ((status & handle_rx) && (chan < priv->plat->rx_queues_to_use)) {
2074                 if (napi_schedule_prep(&ch->rx_napi)) {
2075                         stmmac_disable_dma_irq(priv, priv->ioaddr, chan);
2076                         __napi_schedule_irqoff(&ch->rx_napi);
2077                         status |= handle_tx;
2078                 }
2079         }
2080
2081         if ((status & handle_tx) && (chan < priv->plat->tx_queues_to_use))
2082                 napi_schedule_irqoff(&ch->tx_napi);
2083
2084         return status;
2085 }
2086
2087 /**
2088  * stmmac_dma_interrupt - DMA ISR
2089  * @priv: driver private structure
2090  * Description: this is the DMA ISR. It is called by the main ISR.
2091  * It calls the dwmac dma routine and schedule poll method in case of some
2092  * work can be done.
2093  */
2094 static void stmmac_dma_interrupt(struct stmmac_priv *priv)
2095 {
2096         u32 tx_channel_count = priv->plat->tx_queues_to_use;
2097         u32 rx_channel_count = priv->plat->rx_queues_to_use;
2098         u32 channels_to_check = tx_channel_count > rx_channel_count ?
2099                                 tx_channel_count : rx_channel_count;
2100         u32 chan;
2101         int status[max_t(u32, MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES)];
2102
2103         /* Make sure we never check beyond our status buffer. */
2104         if (WARN_ON_ONCE(channels_to_check > ARRAY_SIZE(status)))
2105                 channels_to_check = ARRAY_SIZE(status);
2106
2107         for (chan = 0; chan < channels_to_check; chan++)
2108                 status[chan] = stmmac_napi_check(priv, chan);
2109
2110         for (chan = 0; chan < tx_channel_count; chan++) {
2111                 if (unlikely(status[chan] & tx_hard_error_bump_tc)) {
2112                         /* Try to bump up the dma threshold on this failure */
2113                         if (unlikely(priv->xstats.threshold != SF_DMA_MODE) &&
2114                             (tc <= 256)) {
2115                                 tc += 64;
2116                                 if (priv->plat->force_thresh_dma_mode)
2117                                         stmmac_set_dma_operation_mode(priv,
2118                                                                       tc,
2119                                                                       tc,
2120                                                                       chan);
2121                                 else
2122                                         stmmac_set_dma_operation_mode(priv,
2123                                                                     tc,
2124                                                                     SF_DMA_MODE,
2125                                                                     chan);
2126                                 priv->xstats.threshold = tc;
2127                         }
2128                 } else if (unlikely(status[chan] == tx_hard_error)) {
2129                         stmmac_tx_err(priv, chan);
2130                 }
2131         }
2132 }
2133
2134 /**
2135  * stmmac_mmc_setup: setup the Mac Management Counters (MMC)
2136  * @priv: driver private structure
2137  * Description: this masks the MMC irq, in fact, the counters are managed in SW.
2138  */
2139 static void stmmac_mmc_setup(struct stmmac_priv *priv)
2140 {
2141         unsigned int mode = MMC_CNTRL_RESET_ON_READ | MMC_CNTRL_COUNTER_RESET |
2142                             MMC_CNTRL_PRESET | MMC_CNTRL_FULL_HALF_PRESET;
2143
2144         stmmac_mmc_intr_all_mask(priv, priv->mmcaddr);
2145
2146         if (priv->dma_cap.rmon) {
2147                 stmmac_mmc_ctrl(priv, priv->mmcaddr, mode);
2148                 memset(&priv->mmc, 0, sizeof(struct stmmac_counters));
2149         } else
2150                 netdev_info(priv->dev, "No MAC Management Counters available\n");
2151 }
2152
2153 /**
2154  * stmmac_get_hw_features - get MAC capabilities from the HW cap. register.
2155  * @priv: driver private structure
2156  * Description:
2157  *  new GMAC chip generations have a new register to indicate the
2158  *  presence of the optional feature/functions.
2159  *  This can be also used to override the value passed through the
2160  *  platform and necessary for old MAC10/100 and GMAC chips.
2161  */
2162 static int stmmac_get_hw_features(struct stmmac_priv *priv)
2163 {
2164         return stmmac_get_hw_feature(priv, priv->ioaddr, &priv->dma_cap) == 0;
2165 }
2166
2167 /**
2168  * stmmac_check_ether_addr - check if the MAC addr is valid
2169  * @priv: driver private structure
2170  * Description:
2171  * it is to verify if the MAC address is valid, in case of failures it
2172  * generates a random MAC address
2173  */
2174 static void stmmac_check_ether_addr(struct stmmac_priv *priv)
2175 {
2176         if (!is_valid_ether_addr(priv->dev->dev_addr)) {
2177                 stmmac_get_umac_addr(priv, priv->hw, priv->dev->dev_addr, 0);
2178                 if (!is_valid_ether_addr(priv->dev->dev_addr))
2179                         eth_hw_addr_random(priv->dev);
2180                 dev_info(priv->device, "device MAC address %pM\n",
2181                          priv->dev->dev_addr);
2182         }
2183 }
2184
2185 /**
2186  * stmmac_init_dma_engine - DMA init.
2187  * @priv: driver private structure
2188  * Description:
2189  * It inits the DMA invoking the specific MAC/GMAC callback.
2190  * Some DMA parameters can be passed from the platform;
2191  * in case of these are not passed a default is kept for the MAC or GMAC.
2192  */
2193 static int stmmac_init_dma_engine(struct stmmac_priv *priv)
2194 {
2195         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2196         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2197         u32 dma_csr_ch = max(rx_channels_count, tx_channels_count);
2198         struct stmmac_rx_queue *rx_q;
2199         struct stmmac_tx_queue *tx_q;
2200         u32 chan = 0;
2201         int atds = 0;
2202         int ret = 0;
2203
2204         if (!priv->plat->dma_cfg || !priv->plat->dma_cfg->pbl) {
2205                 dev_err(priv->device, "Invalid DMA configuration\n");
2206                 return -EINVAL;
2207         }
2208
2209         if (priv->extend_desc && (priv->mode == STMMAC_RING_MODE))
2210                 atds = 1;
2211
2212         ret = stmmac_reset(priv, priv->ioaddr);
2213         if (ret) {
2214                 dev_err(priv->device, "Failed to reset the dma\n");
2215                 return ret;
2216         }
2217
2218         /* DMA Configuration */
2219         stmmac_dma_init(priv, priv->ioaddr, priv->plat->dma_cfg, atds);
2220
2221         if (priv->plat->axi)
2222                 stmmac_axi(priv, priv->ioaddr, priv->plat->axi);
2223
2224         /* DMA CSR Channel configuration */
2225         for (chan = 0; chan < dma_csr_ch; chan++)
2226                 stmmac_init_chan(priv, priv->ioaddr, priv->plat->dma_cfg, chan);
2227
2228         /* DMA RX Channel Configuration */
2229         for (chan = 0; chan < rx_channels_count; chan++) {
2230                 rx_q = &priv->rx_queue[chan];
2231
2232                 stmmac_init_rx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2233                                     rx_q->dma_rx_phy, chan);
2234
2235                 rx_q->rx_tail_addr = rx_q->dma_rx_phy +
2236                             (DMA_RX_SIZE * sizeof(struct dma_desc));
2237                 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
2238                                        rx_q->rx_tail_addr, chan);
2239         }
2240
2241         /* DMA TX Channel Configuration */
2242         for (chan = 0; chan < tx_channels_count; chan++) {
2243                 tx_q = &priv->tx_queue[chan];
2244
2245                 stmmac_init_tx_chan(priv, priv->ioaddr, priv->plat->dma_cfg,
2246                                     tx_q->dma_tx_phy, chan);
2247
2248                 tx_q->tx_tail_addr = tx_q->dma_tx_phy;
2249                 stmmac_set_tx_tail_ptr(priv, priv->ioaddr,
2250                                        tx_q->tx_tail_addr, chan);
2251         }
2252
2253         return ret;
2254 }
2255
2256 static void stmmac_tx_timer_arm(struct stmmac_priv *priv, u32 queue)
2257 {
2258         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2259
2260         mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(priv->tx_coal_timer));
2261 }
2262
2263 /**
2264  * stmmac_tx_timer - mitigation sw timer for tx.
2265  * @data: data pointer
2266  * Description:
2267  * This is the timer handler to directly invoke the stmmac_tx_clean.
2268  */
2269 static void stmmac_tx_timer(struct timer_list *t)
2270 {
2271         struct stmmac_tx_queue *tx_q = from_timer(tx_q, t, txtimer);
2272         struct stmmac_priv *priv = tx_q->priv_data;
2273         struct stmmac_channel *ch;
2274
2275         ch = &priv->channel[tx_q->queue_index];
2276
2277         /*
2278          * If NAPI is already running we can miss some events. Let's rearm
2279          * the timer and try again.
2280          */
2281         if (likely(napi_schedule_prep(&ch->tx_napi)))
2282                 __napi_schedule(&ch->tx_napi);
2283         else
2284                 mod_timer(&tx_q->txtimer, STMMAC_COAL_TIMER(10));
2285 }
2286
2287 /**
2288  * stmmac_init_coalesce - init mitigation options.
2289  * @priv: driver private structure
2290  * Description:
2291  * This inits the coalesce parameters: i.e. timer rate,
2292  * timer handler and default threshold used for enabling the
2293  * interrupt on completion bit.
2294  */
2295 static void stmmac_init_coalesce(struct stmmac_priv *priv)
2296 {
2297         u32 tx_channel_count = priv->plat->tx_queues_to_use;
2298         u32 chan;
2299
2300         priv->tx_coal_frames = STMMAC_TX_FRAMES;
2301         priv->tx_coal_timer = STMMAC_COAL_TX_TIMER;
2302         priv->rx_coal_frames = STMMAC_RX_FRAMES;
2303
2304         for (chan = 0; chan < tx_channel_count; chan++) {
2305                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[chan];
2306
2307                 timer_setup(&tx_q->txtimer, stmmac_tx_timer, 0);
2308         }
2309 }
2310
2311 static void stmmac_set_rings_length(struct stmmac_priv *priv)
2312 {
2313         u32 rx_channels_count = priv->plat->rx_queues_to_use;
2314         u32 tx_channels_count = priv->plat->tx_queues_to_use;
2315         u32 chan;
2316
2317         /* set TX ring length */
2318         for (chan = 0; chan < tx_channels_count; chan++)
2319                 stmmac_set_tx_ring_len(priv, priv->ioaddr,
2320                                 (DMA_TX_SIZE - 1), chan);
2321
2322         /* set RX ring length */
2323         for (chan = 0; chan < rx_channels_count; chan++)
2324                 stmmac_set_rx_ring_len(priv, priv->ioaddr,
2325                                 (DMA_RX_SIZE - 1), chan);
2326 }
2327
2328 /**
2329  *  stmmac_set_tx_queue_weight - Set TX queue weight
2330  *  @priv: driver private structure
2331  *  Description: It is used for setting TX queues weight
2332  */
2333 static void stmmac_set_tx_queue_weight(struct stmmac_priv *priv)
2334 {
2335         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2336         u32 weight;
2337         u32 queue;
2338
2339         for (queue = 0; queue < tx_queues_count; queue++) {
2340                 weight = priv->plat->tx_queues_cfg[queue].weight;
2341                 stmmac_set_mtl_tx_queue_weight(priv, priv->hw, weight, queue);
2342         }
2343 }
2344
2345 /**
2346  *  stmmac_configure_cbs - Configure CBS in TX queue
2347  *  @priv: driver private structure
2348  *  Description: It is used for configuring CBS in AVB TX queues
2349  */
2350 static void stmmac_configure_cbs(struct stmmac_priv *priv)
2351 {
2352         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2353         u32 mode_to_use;
2354         u32 queue;
2355
2356         /* queue 0 is reserved for legacy traffic */
2357         for (queue = 1; queue < tx_queues_count; queue++) {
2358                 mode_to_use = priv->plat->tx_queues_cfg[queue].mode_to_use;
2359                 if (mode_to_use == MTL_QUEUE_DCB)
2360                         continue;
2361
2362                 stmmac_config_cbs(priv, priv->hw,
2363                                 priv->plat->tx_queues_cfg[queue].send_slope,
2364                                 priv->plat->tx_queues_cfg[queue].idle_slope,
2365                                 priv->plat->tx_queues_cfg[queue].high_credit,
2366                                 priv->plat->tx_queues_cfg[queue].low_credit,
2367                                 queue);
2368         }
2369 }
2370
2371 /**
2372  *  stmmac_rx_queue_dma_chan_map - Map RX queue to RX dma channel
2373  *  @priv: driver private structure
2374  *  Description: It is used for mapping RX queues to RX dma channels
2375  */
2376 static void stmmac_rx_queue_dma_chan_map(struct stmmac_priv *priv)
2377 {
2378         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2379         u32 queue;
2380         u32 chan;
2381
2382         for (queue = 0; queue < rx_queues_count; queue++) {
2383                 chan = priv->plat->rx_queues_cfg[queue].chan;
2384                 stmmac_map_mtl_to_dma(priv, priv->hw, queue, chan);
2385         }
2386 }
2387
2388 /**
2389  *  stmmac_mac_config_rx_queues_prio - Configure RX Queue priority
2390  *  @priv: driver private structure
2391  *  Description: It is used for configuring the RX Queue Priority
2392  */
2393 static void stmmac_mac_config_rx_queues_prio(struct stmmac_priv *priv)
2394 {
2395         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2396         u32 queue;
2397         u32 prio;
2398
2399         for (queue = 0; queue < rx_queues_count; queue++) {
2400                 if (!priv->plat->rx_queues_cfg[queue].use_prio)
2401                         continue;
2402
2403                 prio = priv->plat->rx_queues_cfg[queue].prio;
2404                 stmmac_rx_queue_prio(priv, priv->hw, prio, queue);
2405         }
2406 }
2407
2408 /**
2409  *  stmmac_mac_config_tx_queues_prio - Configure TX Queue priority
2410  *  @priv: driver private structure
2411  *  Description: It is used for configuring the TX Queue Priority
2412  */
2413 static void stmmac_mac_config_tx_queues_prio(struct stmmac_priv *priv)
2414 {
2415         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2416         u32 queue;
2417         u32 prio;
2418
2419         for (queue = 0; queue < tx_queues_count; queue++) {
2420                 if (!priv->plat->tx_queues_cfg[queue].use_prio)
2421                         continue;
2422
2423                 prio = priv->plat->tx_queues_cfg[queue].prio;
2424                 stmmac_tx_queue_prio(priv, priv->hw, prio, queue);
2425         }
2426 }
2427
2428 /**
2429  *  stmmac_mac_config_rx_queues_routing - Configure RX Queue Routing
2430  *  @priv: driver private structure
2431  *  Description: It is used for configuring the RX queue routing
2432  */
2433 static void stmmac_mac_config_rx_queues_routing(struct stmmac_priv *priv)
2434 {
2435         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2436         u32 queue;
2437         u8 packet;
2438
2439         for (queue = 0; queue < rx_queues_count; queue++) {
2440                 /* no specific packet type routing specified for the queue */
2441                 if (priv->plat->rx_queues_cfg[queue].pkt_route == 0x0)
2442                         continue;
2443
2444                 packet = priv->plat->rx_queues_cfg[queue].pkt_route;
2445                 stmmac_rx_queue_routing(priv, priv->hw, packet, queue);
2446         }
2447 }
2448
2449 static void stmmac_mac_config_rss(struct stmmac_priv *priv)
2450 {
2451         if (!priv->dma_cap.rssen || !priv->plat->rss_en) {
2452                 priv->rss.enable = false;
2453                 return;
2454         }
2455
2456         if (priv->dev->features & NETIF_F_RXHASH)
2457                 priv->rss.enable = true;
2458         else
2459                 priv->rss.enable = false;
2460
2461         stmmac_rss_configure(priv, priv->hw, &priv->rss,
2462                              priv->plat->rx_queues_to_use);
2463 }
2464
2465 /**
2466  *  stmmac_mtl_configuration - Configure MTL
2467  *  @priv: driver private structure
2468  *  Description: It is used for configurring MTL
2469  */
2470 static void stmmac_mtl_configuration(struct stmmac_priv *priv)
2471 {
2472         u32 rx_queues_count = priv->plat->rx_queues_to_use;
2473         u32 tx_queues_count = priv->plat->tx_queues_to_use;
2474
2475         if (tx_queues_count > 1)
2476                 stmmac_set_tx_queue_weight(priv);
2477
2478         /* Configure MTL RX algorithms */
2479         if (rx_queues_count > 1)
2480                 stmmac_prog_mtl_rx_algorithms(priv, priv->hw,
2481                                 priv->plat->rx_sched_algorithm);
2482
2483         /* Configure MTL TX algorithms */
2484         if (tx_queues_count > 1)
2485                 stmmac_prog_mtl_tx_algorithms(priv, priv->hw,
2486                                 priv->plat->tx_sched_algorithm);
2487
2488         /* Configure CBS in AVB TX queues */
2489         if (tx_queues_count > 1)
2490                 stmmac_configure_cbs(priv);
2491
2492         /* Map RX MTL to DMA channels */
2493         stmmac_rx_queue_dma_chan_map(priv);
2494
2495         /* Enable MAC RX Queues */
2496         stmmac_mac_enable_rx_queues(priv);
2497
2498         /* Set RX priorities */
2499         if (rx_queues_count > 1)
2500                 stmmac_mac_config_rx_queues_prio(priv);
2501
2502         /* Set TX priorities */
2503         if (tx_queues_count > 1)
2504                 stmmac_mac_config_tx_queues_prio(priv);
2505
2506         /* Set RX routing */
2507         if (rx_queues_count > 1)
2508                 stmmac_mac_config_rx_queues_routing(priv);
2509
2510         /* Receive Side Scaling */
2511         if (rx_queues_count > 1)
2512                 stmmac_mac_config_rss(priv);
2513 }
2514
2515 static void stmmac_safety_feat_configuration(struct stmmac_priv *priv)
2516 {
2517         if (priv->dma_cap.asp) {
2518                 netdev_info(priv->dev, "Enabling Safety Features\n");
2519                 stmmac_safety_feat_config(priv, priv->ioaddr, priv->dma_cap.asp);
2520         } else {
2521                 netdev_info(priv->dev, "No Safety Features support found\n");
2522         }
2523 }
2524
2525 /**
2526  * stmmac_hw_setup - setup mac in a usable state.
2527  *  @dev : pointer to the device structure.
2528  *  Description:
2529  *  this is the main function to setup the HW in a usable state because the
2530  *  dma engine is reset, the core registers are configured (e.g. AXI,
2531  *  Checksum features, timers). The DMA is ready to start receiving and
2532  *  transmitting.
2533  *  Return value:
2534  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2535  *  file on failure.
2536  */
2537 static int stmmac_hw_setup(struct net_device *dev, bool init_ptp)
2538 {
2539         struct stmmac_priv *priv = netdev_priv(dev);
2540         u32 rx_cnt = priv->plat->rx_queues_to_use;
2541         u32 tx_cnt = priv->plat->tx_queues_to_use;
2542         u32 chan;
2543         int ret;
2544
2545         /* DMA initialization and SW reset */
2546         ret = stmmac_init_dma_engine(priv);
2547         if (ret < 0) {
2548                 netdev_err(priv->dev, "%s: DMA engine initialization failed\n",
2549                            __func__);
2550                 return ret;
2551         }
2552
2553         /* Copy the MAC addr into the HW  */
2554         stmmac_set_umac_addr(priv, priv->hw, dev->dev_addr, 0);
2555
2556         /* PS and related bits will be programmed according to the speed */
2557         if (priv->hw->pcs) {
2558                 int speed = priv->plat->mac_port_sel_speed;
2559
2560                 if ((speed == SPEED_10) || (speed == SPEED_100) ||
2561                     (speed == SPEED_1000)) {
2562                         priv->hw->ps = speed;
2563                 } else {
2564                         dev_warn(priv->device, "invalid port speed\n");
2565                         priv->hw->ps = 0;
2566                 }
2567         }
2568
2569         /* Initialize the MAC Core */
2570         stmmac_core_init(priv, priv->hw, dev);
2571
2572         /* Initialize MTL*/
2573         stmmac_mtl_configuration(priv);
2574
2575         /* Initialize Safety Features */
2576         stmmac_safety_feat_configuration(priv);
2577
2578         ret = stmmac_rx_ipc(priv, priv->hw);
2579         if (!ret) {
2580                 netdev_warn(priv->dev, "RX IPC Checksum Offload disabled\n");
2581                 priv->plat->rx_coe = STMMAC_RX_COE_NONE;
2582                 priv->hw->rx_csum = 0;
2583         }
2584
2585         /* Enable the MAC Rx/Tx */
2586         stmmac_mac_set(priv, priv->ioaddr, true);
2587
2588         /* Set the HW DMA mode and the COE */
2589         stmmac_dma_operation_mode(priv);
2590
2591         stmmac_mmc_setup(priv);
2592
2593         if (init_ptp) {
2594                 ret = clk_prepare_enable(priv->plat->clk_ptp_ref);
2595                 if (ret < 0)
2596                         netdev_warn(priv->dev, "failed to enable PTP reference clock: %d\n", ret);
2597
2598                 ret = stmmac_init_ptp(priv);
2599                 if (ret == -EOPNOTSUPP)
2600                         netdev_warn(priv->dev, "PTP not supported by HW\n");
2601                 else if (ret)
2602                         netdev_warn(priv->dev, "PTP init failed\n");
2603         }
2604
2605         priv->tx_lpi_timer = STMMAC_DEFAULT_TWT_LS;
2606
2607         if (priv->use_riwt) {
2608                 ret = stmmac_rx_watchdog(priv, priv->ioaddr, MIN_DMA_RIWT, rx_cnt);
2609                 if (!ret)
2610                         priv->rx_riwt = MIN_DMA_RIWT;
2611         }
2612
2613         if (priv->hw->pcs)
2614                 stmmac_pcs_ctrl_ane(priv, priv->ioaddr, 1, priv->hw->ps, 0);
2615
2616         /* set TX and RX rings length */
2617         stmmac_set_rings_length(priv);
2618
2619         /* Enable TSO */
2620         if (priv->tso) {
2621                 for (chan = 0; chan < tx_cnt; chan++)
2622                         stmmac_enable_tso(priv, priv->ioaddr, 1, chan);
2623         }
2624
2625         /* Enable Split Header */
2626         if (priv->sph && priv->hw->rx_csum) {
2627                 for (chan = 0; chan < rx_cnt; chan++)
2628                         stmmac_enable_sph(priv, priv->ioaddr, 1, chan);
2629         }
2630
2631         /* VLAN Tag Insertion */
2632         if (priv->dma_cap.vlins)
2633                 stmmac_enable_vlan(priv, priv->hw, STMMAC_VLAN_INSERT);
2634
2635         /* Start the ball rolling... */
2636         stmmac_start_all_dma(priv);
2637
2638         return 0;
2639 }
2640
2641 static void stmmac_hw_teardown(struct net_device *dev)
2642 {
2643         struct stmmac_priv *priv = netdev_priv(dev);
2644
2645         clk_disable_unprepare(priv->plat->clk_ptp_ref);
2646 }
2647
2648 /**
2649  *  stmmac_open - open entry point of the driver
2650  *  @dev : pointer to the device structure.
2651  *  Description:
2652  *  This function is the open entry point of the driver.
2653  *  Return value:
2654  *  0 on success and an appropriate (-)ve integer as defined in errno.h
2655  *  file on failure.
2656  */
2657 static int stmmac_open(struct net_device *dev)
2658 {
2659         struct stmmac_priv *priv = netdev_priv(dev);
2660         u32 chan;
2661         int ret;
2662
2663         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
2664             priv->hw->pcs != STMMAC_PCS_TBI &&
2665             priv->hw->pcs != STMMAC_PCS_RTBI) {
2666                 ret = stmmac_init_phy(dev);
2667                 if (ret) {
2668                         netdev_err(priv->dev,
2669                                    "%s: Cannot attach to PHY (error: %d)\n",
2670                                    __func__, ret);
2671                         return ret;
2672                 }
2673         }
2674
2675         /* Extra statistics */
2676         memset(&priv->xstats, 0, sizeof(struct stmmac_extra_stats));
2677         priv->xstats.threshold = tc;
2678
2679         priv->dma_buf_sz = STMMAC_ALIGN(buf_sz);
2680         priv->rx_copybreak = STMMAC_RX_COPYBREAK;
2681
2682         ret = alloc_dma_desc_resources(priv);
2683         if (ret < 0) {
2684                 netdev_err(priv->dev, "%s: DMA descriptors allocation failed\n",
2685                            __func__);
2686                 goto dma_desc_error;
2687         }
2688
2689         ret = init_dma_desc_rings(dev, GFP_KERNEL);
2690         if (ret < 0) {
2691                 netdev_err(priv->dev, "%s: DMA descriptors initialization failed\n",
2692                            __func__);
2693                 goto init_error;
2694         }
2695
2696         ret = stmmac_hw_setup(dev, true);
2697         if (ret < 0) {
2698                 netdev_err(priv->dev, "%s: Hw setup failed\n", __func__);
2699                 goto init_error;
2700         }
2701
2702         stmmac_init_coalesce(priv);
2703
2704         phylink_start(priv->phylink);
2705
2706         /* Request the IRQ lines */
2707         ret = request_irq(dev->irq, stmmac_interrupt,
2708                           IRQF_SHARED, dev->name, dev);
2709         if (unlikely(ret < 0)) {
2710                 netdev_err(priv->dev,
2711                            "%s: ERROR: allocating the IRQ %d (error: %d)\n",
2712                            __func__, dev->irq, ret);
2713                 goto irq_error;
2714         }
2715
2716         /* Request the Wake IRQ in case of another line is used for WoL */
2717         if (priv->wol_irq != dev->irq) {
2718                 ret = request_irq(priv->wol_irq, stmmac_interrupt,
2719                                   IRQF_SHARED, dev->name, dev);
2720                 if (unlikely(ret < 0)) {
2721                         netdev_err(priv->dev,
2722                                    "%s: ERROR: allocating the WoL IRQ %d (%d)\n",
2723                                    __func__, priv->wol_irq, ret);
2724                         goto wolirq_error;
2725                 }
2726         }
2727
2728         /* Request the IRQ lines */
2729         if (priv->lpi_irq > 0) {
2730                 ret = request_irq(priv->lpi_irq, stmmac_interrupt, IRQF_SHARED,
2731                                   dev->name, dev);
2732                 if (unlikely(ret < 0)) {
2733                         netdev_err(priv->dev,
2734                                    "%s: ERROR: allocating the LPI IRQ %d (%d)\n",
2735                                    __func__, priv->lpi_irq, ret);
2736                         goto lpiirq_error;
2737                 }
2738         }
2739
2740         stmmac_enable_all_queues(priv);
2741         stmmac_start_all_queues(priv);
2742
2743         return 0;
2744
2745 lpiirq_error:
2746         if (priv->wol_irq != dev->irq)
2747                 free_irq(priv->wol_irq, dev);
2748 wolirq_error:
2749         free_irq(dev->irq, dev);
2750 irq_error:
2751         phylink_stop(priv->phylink);
2752
2753         for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2754                 del_timer_sync(&priv->tx_queue[chan].txtimer);
2755
2756         stmmac_hw_teardown(dev);
2757 init_error:
2758         free_dma_desc_resources(priv);
2759 dma_desc_error:
2760         phylink_disconnect_phy(priv->phylink);
2761         return ret;
2762 }
2763
2764 /**
2765  *  stmmac_release - close entry point of the driver
2766  *  @dev : device pointer.
2767  *  Description:
2768  *  This is the stop entry point of the driver.
2769  */
2770 static int stmmac_release(struct net_device *dev)
2771 {
2772         struct stmmac_priv *priv = netdev_priv(dev);
2773         u32 chan;
2774
2775         if (priv->eee_enabled)
2776                 del_timer_sync(&priv->eee_ctrl_timer);
2777
2778         /* Stop and disconnect the PHY */
2779         phylink_stop(priv->phylink);
2780         phylink_disconnect_phy(priv->phylink);
2781
2782         stmmac_stop_all_queues(priv);
2783
2784         stmmac_disable_all_queues(priv);
2785
2786         for (chan = 0; chan < priv->plat->tx_queues_to_use; chan++)
2787                 del_timer_sync(&priv->tx_queue[chan].txtimer);
2788
2789         /* Free the IRQ lines */
2790         free_irq(dev->irq, dev);
2791         if (priv->wol_irq != dev->irq)
2792                 free_irq(priv->wol_irq, dev);
2793         if (priv->lpi_irq > 0)
2794                 free_irq(priv->lpi_irq, dev);
2795
2796         /* Stop TX/RX DMA and clear the descriptors */
2797         stmmac_stop_all_dma(priv);
2798
2799         /* Release and free the Rx/Tx resources */
2800         free_dma_desc_resources(priv);
2801
2802         /* Disable the MAC Rx/Tx */
2803         stmmac_mac_set(priv, priv->ioaddr, false);
2804
2805         netif_carrier_off(dev);
2806
2807         stmmac_release_ptp(priv);
2808
2809         return 0;
2810 }
2811
2812 static bool stmmac_vlan_insert(struct stmmac_priv *priv, struct sk_buff *skb,
2813                                struct stmmac_tx_queue *tx_q)
2814 {
2815         u16 tag = 0x0, inner_tag = 0x0;
2816         u32 inner_type = 0x0;
2817         struct dma_desc *p;
2818
2819         if (!priv->dma_cap.vlins)
2820                 return false;
2821         if (!skb_vlan_tag_present(skb))
2822                 return false;
2823         if (skb->vlan_proto == htons(ETH_P_8021AD)) {
2824                 inner_tag = skb_vlan_tag_get(skb);
2825                 inner_type = STMMAC_VLAN_INSERT;
2826         }
2827
2828         tag = skb_vlan_tag_get(skb);
2829
2830         p = tx_q->dma_tx + tx_q->cur_tx;
2831         if (stmmac_set_desc_vlan_tag(priv, p, tag, inner_tag, inner_type))
2832                 return false;
2833
2834         stmmac_set_tx_owner(priv, p);
2835         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2836         return true;
2837 }
2838
2839 /**
2840  *  stmmac_tso_allocator - close entry point of the driver
2841  *  @priv: driver private structure
2842  *  @des: buffer start address
2843  *  @total_len: total length to fill in descriptors
2844  *  @last_segmant: condition for the last descriptor
2845  *  @queue: TX queue index
2846  *  Description:
2847  *  This function fills descriptor and request new descriptors according to
2848  *  buffer length to fill
2849  */
2850 static void stmmac_tso_allocator(struct stmmac_priv *priv, dma_addr_t des,
2851                                  int total_len, bool last_segment, u32 queue)
2852 {
2853         struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
2854         struct dma_desc *desc;
2855         u32 buff_size;
2856         int tmp_len;
2857
2858         tmp_len = total_len;
2859
2860         while (tmp_len > 0) {
2861                 dma_addr_t curr_addr;
2862
2863                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2864                 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2865                 desc = tx_q->dma_tx + tx_q->cur_tx;
2866
2867                 curr_addr = des + (total_len - tmp_len);
2868                 if (priv->dma_cap.addr64 <= 32)
2869                         desc->des0 = cpu_to_le32(curr_addr);
2870                 else
2871                         stmmac_set_desc_addr(priv, desc, curr_addr);
2872
2873                 buff_size = tmp_len >= TSO_MAX_BUFF_SIZE ?
2874                             TSO_MAX_BUFF_SIZE : tmp_len;
2875
2876                 stmmac_prepare_tso_tx_desc(priv, desc, 0, buff_size,
2877                                 0, 1,
2878                                 (last_segment) && (tmp_len <= TSO_MAX_BUFF_SIZE),
2879                                 0, 0);
2880
2881                 tmp_len -= TSO_MAX_BUFF_SIZE;
2882         }
2883 }
2884
2885 /**
2886  *  stmmac_tso_xmit - Tx entry point of the driver for oversized frames (TSO)
2887  *  @skb : the socket buffer
2888  *  @dev : device pointer
2889  *  Description: this is the transmit function that is called on TSO frames
2890  *  (support available on GMAC4 and newer chips).
2891  *  Diagram below show the ring programming in case of TSO frames:
2892  *
2893  *  First Descriptor
2894  *   --------
2895  *   | DES0 |---> buffer1 = L2/L3/L4 header
2896  *   | DES1 |---> TCP Payload (can continue on next descr...)
2897  *   | DES2 |---> buffer 1 and 2 len
2898  *   | DES3 |---> must set TSE, TCP hdr len-> [22:19]. TCP payload len [17:0]
2899  *   --------
2900  *      |
2901  *     ...
2902  *      |
2903  *   --------
2904  *   | DES0 | --| Split TCP Payload on Buffers 1 and 2
2905  *   | DES1 | --|
2906  *   | DES2 | --> buffer 1 and 2 len
2907  *   | DES3 |
2908  *   --------
2909  *
2910  * mss is fixed when enable tso, so w/o programming the TDES3 ctx field.
2911  */
2912 static netdev_tx_t stmmac_tso_xmit(struct sk_buff *skb, struct net_device *dev)
2913 {
2914         struct dma_desc *desc, *first, *mss_desc = NULL;
2915         struct stmmac_priv *priv = netdev_priv(dev);
2916         int nfrags = skb_shinfo(skb)->nr_frags;
2917         u32 queue = skb_get_queue_mapping(skb);
2918         struct stmmac_tx_queue *tx_q;
2919         unsigned int first_entry;
2920         u8 proto_hdr_len, hdr;
2921         int tmp_pay_len = 0;
2922         u32 pay_len, mss;
2923         dma_addr_t des;
2924         bool has_vlan;
2925         int i;
2926
2927         tx_q = &priv->tx_queue[queue];
2928
2929         /* Compute header lengths */
2930         if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4) {
2931                 proto_hdr_len = skb_transport_offset(skb) + sizeof(struct udphdr);
2932                 hdr = sizeof(struct udphdr);
2933         } else {
2934                 proto_hdr_len = skb_transport_offset(skb) + tcp_hdrlen(skb);
2935                 hdr = tcp_hdrlen(skb);
2936         }
2937
2938         /* Desc availability based on threshold should be enough safe */
2939         if (unlikely(stmmac_tx_avail(priv, queue) <
2940                 (((skb->len - proto_hdr_len) / TSO_MAX_BUFF_SIZE + 1)))) {
2941                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
2942                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
2943                                                                 queue));
2944                         /* This is a hard error, log it. */
2945                         netdev_err(priv->dev,
2946                                    "%s: Tx Ring full when queue awake\n",
2947                                    __func__);
2948                 }
2949                 return NETDEV_TX_BUSY;
2950         }
2951
2952         pay_len = skb_headlen(skb) - proto_hdr_len; /* no frags */
2953
2954         mss = skb_shinfo(skb)->gso_size;
2955
2956         /* set new MSS value if needed */
2957         if (mss != tx_q->mss) {
2958                 mss_desc = tx_q->dma_tx + tx_q->cur_tx;
2959                 stmmac_set_mss(priv, mss_desc, mss);
2960                 tx_q->mss = mss;
2961                 tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
2962                 WARN_ON(tx_q->tx_skbuff[tx_q->cur_tx]);
2963         }
2964
2965         if (netif_msg_tx_queued(priv)) {
2966                 pr_info("%s: hdrlen %d, hdr_len %d, pay_len %d, mss %d\n",
2967                         __func__, hdr, proto_hdr_len, pay_len, mss);
2968                 pr_info("\tskb->len %d, skb->data_len %d\n", skb->len,
2969                         skb->data_len);
2970         }
2971
2972         /* Check if VLAN can be inserted by HW */
2973         has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
2974
2975         first_entry = tx_q->cur_tx;
2976         WARN_ON(tx_q->tx_skbuff[first_entry]);
2977
2978         desc = tx_q->dma_tx + first_entry;
2979         first = desc;
2980
2981         if (has_vlan)
2982                 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
2983
2984         /* first descriptor: fill Headers on Buf1 */
2985         des = dma_map_single(priv->device, skb->data, skb_headlen(skb),
2986                              DMA_TO_DEVICE);
2987         if (dma_mapping_error(priv->device, des))
2988                 goto dma_map_err;
2989
2990         tx_q->tx_skbuff_dma[first_entry].buf = des;
2991         tx_q->tx_skbuff_dma[first_entry].len = skb_headlen(skb);
2992
2993         if (priv->dma_cap.addr64 <= 32) {
2994                 first->des0 = cpu_to_le32(des);
2995
2996                 /* Fill start of payload in buff2 of first descriptor */
2997                 if (pay_len)
2998                         first->des1 = cpu_to_le32(des + proto_hdr_len);
2999
3000                 /* If needed take extra descriptors to fill the remaining payload */
3001                 tmp_pay_len = pay_len - TSO_MAX_BUFF_SIZE;
3002         } else {
3003                 stmmac_set_desc_addr(priv, first, des);
3004                 tmp_pay_len = pay_len;
3005                 des += proto_hdr_len;
3006                 pay_len = 0;
3007         }
3008
3009         stmmac_tso_allocator(priv, des, tmp_pay_len, (nfrags == 0), queue);
3010
3011         /* Prepare fragments */
3012         for (i = 0; i < nfrags; i++) {
3013                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3014
3015                 des = skb_frag_dma_map(priv->device, frag, 0,
3016                                        skb_frag_size(frag),
3017                                        DMA_TO_DEVICE);
3018                 if (dma_mapping_error(priv->device, des))
3019                         goto dma_map_err;
3020
3021                 stmmac_tso_allocator(priv, des, skb_frag_size(frag),
3022                                      (i == nfrags - 1), queue);
3023
3024                 tx_q->tx_skbuff_dma[tx_q->cur_tx].buf = des;
3025                 tx_q->tx_skbuff_dma[tx_q->cur_tx].len = skb_frag_size(frag);
3026                 tx_q->tx_skbuff_dma[tx_q->cur_tx].map_as_page = true;
3027         }
3028
3029         tx_q->tx_skbuff_dma[tx_q->cur_tx].last_segment = true;
3030
3031         /* Only the last descriptor gets to point to the skb. */
3032         tx_q->tx_skbuff[tx_q->cur_tx] = skb;
3033
3034         /* Manage tx mitigation */
3035         tx_q->tx_count_frames += nfrags + 1;
3036         if (likely(priv->tx_coal_frames > tx_q->tx_count_frames) &&
3037             !((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3038               priv->hwts_tx_en)) {
3039                 stmmac_tx_timer_arm(priv, queue);
3040         } else {
3041                 desc = &tx_q->dma_tx[tx_q->cur_tx];
3042                 tx_q->tx_count_frames = 0;
3043                 stmmac_set_tx_ic(priv, desc);
3044                 priv->xstats.tx_set_ic_bit++;
3045         }
3046
3047         /* We've used all descriptors we need for this skb, however,
3048          * advance cur_tx so that it references a fresh descriptor.
3049          * ndo_start_xmit will fill this descriptor the next time it's
3050          * called and stmmac_tx_clean may clean up to this descriptor.
3051          */
3052         tx_q->cur_tx = STMMAC_GET_ENTRY(tx_q->cur_tx, DMA_TX_SIZE);
3053
3054         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3055                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3056                           __func__);
3057                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3058         }
3059
3060         dev->stats.tx_bytes += skb->len;
3061         priv->xstats.tx_tso_frames++;
3062         priv->xstats.tx_tso_nfrags += nfrags;
3063
3064         if (priv->sarc_type)
3065                 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3066
3067         skb_tx_timestamp(skb);
3068
3069         if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3070                      priv->hwts_tx_en)) {
3071                 /* declare that device is doing timestamping */
3072                 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3073                 stmmac_enable_tx_timestamp(priv, first);
3074         }
3075
3076         /* Complete the first descriptor before granting the DMA */
3077         stmmac_prepare_tso_tx_desc(priv, first, 1,
3078                         proto_hdr_len,
3079                         pay_len,
3080                         1, tx_q->tx_skbuff_dma[first_entry].last_segment,
3081                         hdr / 4, (skb->len - proto_hdr_len));
3082
3083         /* If context desc is used to change MSS */
3084         if (mss_desc) {
3085                 /* Make sure that first descriptor has been completely
3086                  * written, including its own bit. This is because MSS is
3087                  * actually before first descriptor, so we need to make
3088                  * sure that MSS's own bit is the last thing written.
3089                  */
3090                 dma_wmb();
3091                 stmmac_set_tx_owner(priv, mss_desc);
3092         }
3093
3094         /* The own bit must be the latest setting done when prepare the
3095          * descriptor and then barrier is needed to make sure that
3096          * all is coherent before granting the DMA engine.
3097          */
3098         wmb();
3099
3100         if (netif_msg_pktdata(priv)) {
3101                 pr_info("%s: curr=%d dirty=%d f=%d, e=%d, f_p=%p, nfrags %d\n",
3102                         __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3103                         tx_q->cur_tx, first, nfrags);
3104
3105                 stmmac_display_ring(priv, (void *)tx_q->dma_tx, DMA_TX_SIZE, 0);
3106
3107                 pr_info(">>> frame to be transmitted: ");
3108                 print_pkt(skb->data, skb_headlen(skb));
3109         }
3110
3111         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3112
3113         tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc));
3114         stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3115
3116         return NETDEV_TX_OK;
3117
3118 dma_map_err:
3119         dev_err(priv->device, "Tx dma map failed\n");
3120         dev_kfree_skb(skb);
3121         priv->dev->stats.tx_dropped++;
3122         return NETDEV_TX_OK;
3123 }
3124
3125 /**
3126  *  stmmac_xmit - Tx entry point of the driver
3127  *  @skb : the socket buffer
3128  *  @dev : device pointer
3129  *  Description : this is the tx entry point of the driver.
3130  *  It programs the chain or the ring and supports oversized frames
3131  *  and SG feature.
3132  */
3133 static netdev_tx_t stmmac_xmit(struct sk_buff *skb, struct net_device *dev)
3134 {
3135         struct stmmac_priv *priv = netdev_priv(dev);
3136         unsigned int nopaged_len = skb_headlen(skb);
3137         int i, csum_insertion = 0, is_jumbo = 0;
3138         u32 queue = skb_get_queue_mapping(skb);
3139         int nfrags = skb_shinfo(skb)->nr_frags;
3140         int gso = skb_shinfo(skb)->gso_type;
3141         struct dma_desc *desc, *first;
3142         struct stmmac_tx_queue *tx_q;
3143         unsigned int first_entry;
3144         unsigned int enh_desc;
3145         dma_addr_t des;
3146         bool has_vlan;
3147         int entry;
3148
3149         tx_q = &priv->tx_queue[queue];
3150
3151         if (priv->tx_path_in_lpi_mode)
3152                 stmmac_disable_eee_mode(priv);
3153
3154         /* Manage oversized TCP frames for GMAC4 device */
3155         if (skb_is_gso(skb) && priv->tso) {
3156                 if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6))
3157                         return stmmac_tso_xmit(skb, dev);
3158                 if (priv->plat->has_gmac4 && (gso & SKB_GSO_UDP_L4))
3159                         return stmmac_tso_xmit(skb, dev);
3160         }
3161
3162         if (unlikely(stmmac_tx_avail(priv, queue) < nfrags + 1)) {
3163                 if (!netif_tx_queue_stopped(netdev_get_tx_queue(dev, queue))) {
3164                         netif_tx_stop_queue(netdev_get_tx_queue(priv->dev,
3165                                                                 queue));
3166                         /* This is a hard error, log it. */
3167                         netdev_err(priv->dev,
3168                                    "%s: Tx Ring full when queue awake\n",
3169                                    __func__);
3170                 }
3171                 return NETDEV_TX_BUSY;
3172         }
3173
3174         /* Check if VLAN can be inserted by HW */
3175         has_vlan = stmmac_vlan_insert(priv, skb, tx_q);
3176
3177         entry = tx_q->cur_tx;
3178         first_entry = entry;
3179         WARN_ON(tx_q->tx_skbuff[first_entry]);
3180
3181         csum_insertion = (skb->ip_summed == CHECKSUM_PARTIAL);
3182
3183         if (likely(priv->extend_desc))
3184                 desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3185         else
3186                 desc = tx_q->dma_tx + entry;
3187
3188         first = desc;
3189
3190         if (has_vlan)
3191                 stmmac_set_desc_vlan(priv, first, STMMAC_VLAN_INSERT);
3192
3193         enh_desc = priv->plat->enh_desc;
3194         /* To program the descriptors according to the size of the frame */
3195         if (enh_desc)
3196                 is_jumbo = stmmac_is_jumbo_frm(priv, skb->len, enh_desc);
3197
3198         if (unlikely(is_jumbo)) {
3199                 entry = stmmac_jumbo_frm(priv, tx_q, skb, csum_insertion);
3200                 if (unlikely(entry < 0) && (entry != -EINVAL))
3201                         goto dma_map_err;
3202         }
3203
3204         for (i = 0; i < nfrags; i++) {
3205                 const skb_frag_t *frag = &skb_shinfo(skb)->frags[i];
3206                 int len = skb_frag_size(frag);
3207                 bool last_segment = (i == (nfrags - 1));
3208
3209                 entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3210                 WARN_ON(tx_q->tx_skbuff[entry]);
3211
3212                 if (likely(priv->extend_desc))
3213                         desc = (struct dma_desc *)(tx_q->dma_etx + entry);
3214                 else
3215                         desc = tx_q->dma_tx + entry;
3216
3217                 des = skb_frag_dma_map(priv->device, frag, 0, len,
3218                                        DMA_TO_DEVICE);
3219                 if (dma_mapping_error(priv->device, des))
3220                         goto dma_map_err; /* should reuse desc w/o issues */
3221
3222                 tx_q->tx_skbuff_dma[entry].buf = des;
3223
3224                 stmmac_set_desc_addr(priv, desc, des);
3225
3226                 tx_q->tx_skbuff_dma[entry].map_as_page = true;
3227                 tx_q->tx_skbuff_dma[entry].len = len;
3228                 tx_q->tx_skbuff_dma[entry].last_segment = last_segment;
3229
3230                 /* Prepare the descriptor and set the own bit too */
3231                 stmmac_prepare_tx_desc(priv, desc, 0, len, csum_insertion,
3232                                 priv->mode, 1, last_segment, skb->len);
3233         }
3234
3235         /* Only the last descriptor gets to point to the skb. */
3236         tx_q->tx_skbuff[entry] = skb;
3237
3238         /* According to the coalesce parameter the IC bit for the latest
3239          * segment is reset and the timer re-started to clean the tx status.
3240          * This approach takes care about the fragments: desc is the first
3241          * element in case of no SG.
3242          */
3243         tx_q->tx_count_frames += nfrags + 1;
3244         if (likely(priv->tx_coal_frames > tx_q->tx_count_frames) &&
3245             !((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3246               priv->hwts_tx_en)) {
3247                 stmmac_tx_timer_arm(priv, queue);
3248         } else {
3249                 if (likely(priv->extend_desc))
3250                         desc = &tx_q->dma_etx[entry].basic;
3251                 else
3252                         desc = &tx_q->dma_tx[entry];
3253
3254                 tx_q->tx_count_frames = 0;
3255                 stmmac_set_tx_ic(priv, desc);
3256                 priv->xstats.tx_set_ic_bit++;
3257         }
3258
3259         /* We've used all descriptors we need for this skb, however,
3260          * advance cur_tx so that it references a fresh descriptor.
3261          * ndo_start_xmit will fill this descriptor the next time it's
3262          * called and stmmac_tx_clean may clean up to this descriptor.
3263          */
3264         entry = STMMAC_GET_ENTRY(entry, DMA_TX_SIZE);
3265         tx_q->cur_tx = entry;
3266
3267         if (netif_msg_pktdata(priv)) {
3268                 void *tx_head;
3269
3270                 netdev_dbg(priv->dev,
3271                            "%s: curr=%d dirty=%d f=%d, e=%d, first=%p, nfrags=%d",
3272                            __func__, tx_q->cur_tx, tx_q->dirty_tx, first_entry,
3273                            entry, first, nfrags);
3274
3275                 if (priv->extend_desc)
3276                         tx_head = (void *)tx_q->dma_etx;
3277                 else
3278                         tx_head = (void *)tx_q->dma_tx;
3279
3280                 stmmac_display_ring(priv, tx_head, DMA_TX_SIZE, false);
3281
3282                 netdev_dbg(priv->dev, ">>> frame to be transmitted: ");
3283                 print_pkt(skb->data, skb->len);
3284         }
3285
3286         if (unlikely(stmmac_tx_avail(priv, queue) <= (MAX_SKB_FRAGS + 1))) {
3287                 netif_dbg(priv, hw, priv->dev, "%s: stop transmitted packets\n",
3288                           __func__);
3289                 netif_tx_stop_queue(netdev_get_tx_queue(priv->dev, queue));
3290         }
3291
3292         dev->stats.tx_bytes += skb->len;
3293
3294         if (priv->sarc_type)
3295                 stmmac_set_desc_sarc(priv, first, priv->sarc_type);
3296
3297         skb_tx_timestamp(skb);
3298
3299         /* Ready to fill the first descriptor and set the OWN bit w/o any
3300          * problems because all the descriptors are actually ready to be
3301          * passed to the DMA engine.
3302          */
3303         if (likely(!is_jumbo)) {
3304                 bool last_segment = (nfrags == 0);
3305
3306                 des = dma_map_single(priv->device, skb->data,
3307                                      nopaged_len, DMA_TO_DEVICE);
3308                 if (dma_mapping_error(priv->device, des))
3309                         goto dma_map_err;
3310
3311                 tx_q->tx_skbuff_dma[first_entry].buf = des;
3312
3313                 stmmac_set_desc_addr(priv, first, des);
3314
3315                 tx_q->tx_skbuff_dma[first_entry].len = nopaged_len;
3316                 tx_q->tx_skbuff_dma[first_entry].last_segment = last_segment;
3317
3318                 if (unlikely((skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP) &&
3319                              priv->hwts_tx_en)) {
3320                         /* declare that device is doing timestamping */
3321                         skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
3322                         stmmac_enable_tx_timestamp(priv, first);
3323                 }
3324
3325                 /* Prepare the first descriptor setting the OWN bit too */
3326                 stmmac_prepare_tx_desc(priv, first, 1, nopaged_len,
3327                                 csum_insertion, priv->mode, 1, last_segment,
3328                                 skb->len);
3329         } else {
3330                 stmmac_set_tx_owner(priv, first);
3331         }
3332
3333         /* The own bit must be the latest setting done when prepare the
3334          * descriptor and then barrier is needed to make sure that
3335          * all is coherent before granting the DMA engine.
3336          */
3337         wmb();
3338
3339         netdev_tx_sent_queue(netdev_get_tx_queue(dev, queue), skb->len);
3340
3341         stmmac_enable_dma_transmission(priv, priv->ioaddr);
3342
3343         tx_q->tx_tail_addr = tx_q->dma_tx_phy + (tx_q->cur_tx * sizeof(*desc));
3344         stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr, queue);
3345
3346         return NETDEV_TX_OK;
3347
3348 dma_map_err:
3349         netdev_err(priv->dev, "Tx DMA map failed\n");
3350         dev_kfree_skb(skb);
3351         priv->dev->stats.tx_dropped++;
3352         return NETDEV_TX_OK;
3353 }
3354
3355 static void stmmac_rx_vlan(struct net_device *dev, struct sk_buff *skb)
3356 {
3357         struct vlan_ethhdr *veth;
3358         __be16 vlan_proto;
3359         u16 vlanid;
3360
3361         veth = (struct vlan_ethhdr *)skb->data;
3362         vlan_proto = veth->h_vlan_proto;
3363
3364         if ((vlan_proto == htons(ETH_P_8021Q) &&
3365              dev->features & NETIF_F_HW_VLAN_CTAG_RX) ||
3366             (vlan_proto == htons(ETH_P_8021AD) &&
3367              dev->features & NETIF_F_HW_VLAN_STAG_RX)) {
3368                 /* pop the vlan tag */
3369                 vlanid = ntohs(veth->h_vlan_TCI);
3370                 memmove(skb->data + VLAN_HLEN, veth, ETH_ALEN * 2);
3371                 skb_pull(skb, VLAN_HLEN);
3372                 __vlan_hwaccel_put_tag(skb, vlan_proto, vlanid);
3373         }
3374 }
3375
3376
3377 static inline int stmmac_rx_threshold_count(struct stmmac_rx_queue *rx_q)
3378 {
3379         if (rx_q->rx_zeroc_thresh < STMMAC_RX_THRESH)
3380                 return 0;
3381
3382         return 1;
3383 }
3384
3385 /**
3386  * stmmac_rx_refill - refill used skb preallocated buffers
3387  * @priv: driver private structure
3388  * @queue: RX queue index
3389  * Description : this is to reallocate the skb for the reception process
3390  * that is based on zero-copy.
3391  */
3392 static inline void stmmac_rx_refill(struct stmmac_priv *priv, u32 queue)
3393 {
3394         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3395         int len, dirty = stmmac_rx_dirty(priv, queue);
3396         unsigned int entry = rx_q->dirty_rx;
3397
3398         len = DIV_ROUND_UP(priv->dma_buf_sz, PAGE_SIZE) * PAGE_SIZE;
3399
3400         while (dirty-- > 0) {
3401                 struct stmmac_rx_buffer *buf = &rx_q->buf_pool[entry];
3402                 struct dma_desc *p;
3403                 bool use_rx_wd;
3404
3405                 if (priv->extend_desc)
3406                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3407                 else
3408                         p = rx_q->dma_rx + entry;
3409
3410                 if (!buf->page) {
3411                         buf->page = page_pool_dev_alloc_pages(rx_q->page_pool);
3412                         if (!buf->page)
3413                                 break;
3414                 }
3415
3416                 if (priv->sph && !buf->sec_page) {
3417                         buf->sec_page = page_pool_dev_alloc_pages(rx_q->page_pool);
3418                         if (!buf->sec_page)
3419                                 break;
3420
3421                         buf->sec_addr = page_pool_get_dma_addr(buf->sec_page);
3422
3423                         dma_sync_single_for_device(priv->device, buf->sec_addr,
3424                                                    len, DMA_FROM_DEVICE);
3425                 }
3426
3427                 buf->addr = page_pool_get_dma_addr(buf->page);
3428
3429                 /* Sync whole allocation to device. This will invalidate old
3430                  * data.
3431                  */
3432                 dma_sync_single_for_device(priv->device, buf->addr, len,
3433                                            DMA_FROM_DEVICE);
3434
3435                 stmmac_set_desc_addr(priv, p, buf->addr);
3436                 stmmac_set_desc_sec_addr(priv, p, buf->sec_addr);
3437                 stmmac_refill_desc3(priv, rx_q, p);
3438
3439                 rx_q->rx_count_frames++;
3440                 rx_q->rx_count_frames += priv->rx_coal_frames;
3441                 if (rx_q->rx_count_frames > priv->rx_coal_frames)
3442                         rx_q->rx_count_frames = 0;
3443
3444                 use_rx_wd = !priv->rx_coal_frames;
3445                 use_rx_wd |= rx_q->rx_count_frames > 0;
3446                 if (!priv->use_riwt)
3447                         use_rx_wd = false;
3448
3449                 dma_wmb();
3450                 stmmac_set_rx_owner(priv, p, use_rx_wd);
3451
3452                 entry = STMMAC_GET_ENTRY(entry, DMA_RX_SIZE);
3453         }
3454         rx_q->dirty_rx = entry;
3455         rx_q->rx_tail_addr = rx_q->dma_rx_phy +
3456                             (rx_q->dirty_rx * sizeof(struct dma_desc));
3457         stmmac_set_rx_tail_ptr(priv, priv->ioaddr, rx_q->rx_tail_addr, queue);
3458 }
3459
3460 static unsigned int stmmac_rx_buf1_len(struct stmmac_priv *priv,
3461                                        struct dma_desc *p,
3462                                        int status, unsigned int len)
3463 {
3464         int ret, coe = priv->hw->rx_csum;
3465         unsigned int plen = 0, hlen = 0;
3466
3467         /* Not first descriptor, buffer is always zero */
3468         if (priv->sph && len)
3469                 return 0;
3470
3471         /* First descriptor, get split header length */
3472         ret = stmmac_get_rx_header_len(priv, p, &hlen);
3473         if (priv->sph && hlen) {
3474                 priv->xstats.rx_split_hdr_pkt_n++;
3475                 return hlen;
3476         }
3477
3478         /* First descriptor, not last descriptor and not split header */
3479         if (status & rx_not_ls)
3480                 return priv->dma_buf_sz;
3481
3482         plen = stmmac_get_rx_frame_len(priv, p, coe);
3483
3484         /* First descriptor and last descriptor and not split header */
3485         return min_t(unsigned int, priv->dma_buf_sz, plen);
3486 }
3487
3488 static unsigned int stmmac_rx_buf2_len(struct stmmac_priv *priv,
3489                                        struct dma_desc *p,
3490                                        int status, unsigned int len)
3491 {
3492         int coe = priv->hw->rx_csum;
3493         unsigned int plen = 0;
3494
3495         /* Not split header, buffer is not available */
3496         if (!priv->sph)
3497                 return 0;
3498
3499         /* Not last descriptor */
3500         if (status & rx_not_ls)
3501                 return priv->dma_buf_sz;
3502
3503         plen = stmmac_get_rx_frame_len(priv, p, coe);
3504
3505         /* Last descriptor */
3506         return plen - len;
3507 }
3508
3509 /**
3510  * stmmac_rx - manage the receive process
3511  * @priv: driver private structure
3512  * @limit: napi bugget
3513  * @queue: RX queue index.
3514  * Description :  this the function called by the napi poll method.
3515  * It gets all the frames inside the ring.
3516  */
3517 static int stmmac_rx(struct stmmac_priv *priv, int limit, u32 queue)
3518 {
3519         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3520         struct stmmac_channel *ch = &priv->channel[queue];
3521         unsigned int count = 0, error = 0, len = 0;
3522         int status = 0, coe = priv->hw->rx_csum;
3523         unsigned int next_entry = rx_q->cur_rx;
3524         struct sk_buff *skb = NULL;
3525
3526         if (netif_msg_rx_status(priv)) {
3527                 void *rx_head;
3528
3529                 netdev_dbg(priv->dev, "%s: descriptor ring:\n", __func__);
3530                 if (priv->extend_desc)
3531                         rx_head = (void *)rx_q->dma_erx;
3532                 else
3533                         rx_head = (void *)rx_q->dma_rx;
3534
3535                 stmmac_display_ring(priv, rx_head, DMA_RX_SIZE, true);
3536         }
3537         while (count < limit) {
3538                 unsigned int buf1_len = 0, buf2_len = 0;
3539                 enum pkt_hash_types hash_type;
3540                 struct stmmac_rx_buffer *buf;
3541                 struct dma_desc *np, *p;
3542                 int entry;
3543                 u32 hash;
3544
3545                 if (!count && rx_q->state_saved) {
3546                         skb = rx_q->state.skb;
3547                         error = rx_q->state.error;
3548                         len = rx_q->state.len;
3549                 } else {
3550                         rx_q->state_saved = false;
3551                         skb = NULL;
3552                         error = 0;
3553                         len = 0;
3554                 }
3555
3556                 if (count >= limit)
3557                         break;
3558
3559 read_again:
3560                 buf1_len = 0;
3561                 buf2_len = 0;
3562                 entry = next_entry;
3563                 buf = &rx_q->buf_pool[entry];
3564
3565                 if (priv->extend_desc)
3566                         p = (struct dma_desc *)(rx_q->dma_erx + entry);
3567                 else
3568                         p = rx_q->dma_rx + entry;
3569
3570                 /* read the status of the incoming frame */
3571                 status = stmmac_rx_status(priv, &priv->dev->stats,
3572                                 &priv->xstats, p);
3573                 /* check if managed by the DMA otherwise go ahead */
3574                 if (unlikely(status & dma_own))
3575                         break;
3576
3577                 rx_q->cur_rx = STMMAC_GET_ENTRY(rx_q->cur_rx, DMA_RX_SIZE);
3578                 next_entry = rx_q->cur_rx;
3579
3580                 if (priv->extend_desc)
3581                         np = (struct dma_desc *)(rx_q->dma_erx + next_entry);
3582                 else
3583                         np = rx_q->dma_rx + next_entry;
3584
3585                 prefetch(np);
3586
3587                 if (priv->extend_desc)
3588                         stmmac_rx_extended_status(priv, &priv->dev->stats,
3589                                         &priv->xstats, rx_q->dma_erx + entry);
3590                 if (unlikely(status == discard_frame)) {
3591                         page_pool_recycle_direct(rx_q->page_pool, buf->page);
3592                         buf->page = NULL;
3593                         error = 1;
3594                         if (!priv->hwts_rx_en)
3595                                 priv->dev->stats.rx_errors++;
3596                 }
3597
3598                 if (unlikely(error && (status & rx_not_ls)))
3599                         goto read_again;
3600                 if (unlikely(error)) {
3601                         dev_kfree_skb(skb);
3602                         skb = NULL;
3603                         count++;
3604                         continue;
3605                 }
3606
3607                 /* Buffer is good. Go on. */
3608
3609                 prefetch(page_address(buf->page));
3610                 if (buf->sec_page)
3611                         prefetch(page_address(buf->sec_page));
3612
3613                 buf1_len = stmmac_rx_buf1_len(priv, p, status, len);
3614                 len += buf1_len;
3615                 buf2_len = stmmac_rx_buf2_len(priv, p, status, len);
3616                 len += buf2_len;
3617
3618                 /* ACS is set; GMAC core strips PAD/FCS for IEEE 802.3
3619                  * Type frames (LLC/LLC-SNAP)
3620                  *
3621                  * llc_snap is never checked in GMAC >= 4, so this ACS
3622                  * feature is always disabled and packets need to be
3623                  * stripped manually.
3624                  */
3625                 if (unlikely(priv->synopsys_id >= DWMAC_CORE_4_00) ||
3626                     unlikely(status != llc_snap)) {
3627                         if (buf2_len)
3628                                 buf2_len -= ETH_FCS_LEN;
3629                         else
3630                                 buf1_len -= ETH_FCS_LEN;
3631
3632                         len -= ETH_FCS_LEN;
3633                 }
3634
3635                 if (!skb) {
3636                         skb = napi_alloc_skb(&ch->rx_napi, buf1_len);
3637                         if (!skb) {
3638                                 priv->dev->stats.rx_dropped++;
3639                                 count++;
3640                                 goto drain_data;
3641                         }
3642
3643                         dma_sync_single_for_cpu(priv->device, buf->addr,
3644                                                 buf1_len, DMA_FROM_DEVICE);
3645                         skb_copy_to_linear_data(skb, page_address(buf->page),
3646                                                 buf1_len);
3647                         skb_put(skb, buf1_len);
3648
3649                         /* Data payload copied into SKB, page ready for recycle */
3650                         page_pool_recycle_direct(rx_q->page_pool, buf->page);
3651                         buf->page = NULL;
3652                 } else if (buf1_len) {
3653                         dma_sync_single_for_cpu(priv->device, buf->addr,
3654                                                 buf1_len, DMA_FROM_DEVICE);
3655                         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3656                                         buf->page, 0, buf1_len,
3657                                         priv->dma_buf_sz);
3658
3659                         /* Data payload appended into SKB */
3660                         page_pool_release_page(rx_q->page_pool, buf->page);
3661                         buf->page = NULL;
3662                 }
3663
3664                 if (buf2_len) {
3665                         dma_sync_single_for_cpu(priv->device, buf->sec_addr,
3666                                                 buf2_len, DMA_FROM_DEVICE);
3667                         skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags,
3668                                         buf->sec_page, 0, buf2_len,
3669                                         priv->dma_buf_sz);
3670
3671                         /* Data payload appended into SKB */
3672                         page_pool_release_page(rx_q->page_pool, buf->sec_page);
3673                         buf->sec_page = NULL;
3674                 }
3675
3676 drain_data:
3677                 if (likely(status & rx_not_ls))
3678                         goto read_again;
3679                 if (!skb)
3680                         continue;
3681
3682                 /* Got entire packet into SKB. Finish it. */
3683
3684                 stmmac_get_rx_hwtstamp(priv, p, np, skb);
3685                 stmmac_rx_vlan(priv->dev, skb);
3686                 skb->protocol = eth_type_trans(skb, priv->dev);
3687
3688                 if (unlikely(!coe))
3689                         skb_checksum_none_assert(skb);
3690                 else
3691                         skb->ip_summed = CHECKSUM_UNNECESSARY;
3692
3693                 if (!stmmac_get_rx_hash(priv, p, &hash, &hash_type))
3694                         skb_set_hash(skb, hash, hash_type);
3695
3696                 skb_record_rx_queue(skb, queue);
3697                 napi_gro_receive(&ch->rx_napi, skb);
3698                 skb = NULL;
3699
3700                 priv->dev->stats.rx_packets++;
3701                 priv->dev->stats.rx_bytes += len;
3702                 count++;
3703         }
3704
3705         if (status & rx_not_ls || skb) {
3706                 rx_q->state_saved = true;
3707                 rx_q->state.skb = skb;
3708                 rx_q->state.error = error;
3709                 rx_q->state.len = len;
3710         }
3711
3712         stmmac_rx_refill(priv, queue);
3713
3714         priv->xstats.rx_pkt_n += count;
3715
3716         return count;
3717 }
3718
3719 static int stmmac_napi_poll_rx(struct napi_struct *napi, int budget)
3720 {
3721         struct stmmac_channel *ch =
3722                 container_of(napi, struct stmmac_channel, rx_napi);
3723         struct stmmac_priv *priv = ch->priv_data;
3724         u32 chan = ch->index;
3725         int work_done;
3726
3727         priv->xstats.napi_poll++;
3728
3729         work_done = stmmac_rx(priv, budget, chan);
3730         if (work_done < budget && napi_complete_done(napi, work_done))
3731                 stmmac_enable_dma_irq(priv, priv->ioaddr, chan);
3732         return work_done;
3733 }
3734
3735 static int stmmac_napi_poll_tx(struct napi_struct *napi, int budget)
3736 {
3737         struct stmmac_channel *ch =
3738                 container_of(napi, struct stmmac_channel, tx_napi);
3739         struct stmmac_priv *priv = ch->priv_data;
3740         struct stmmac_tx_queue *tx_q;
3741         u32 chan = ch->index;
3742         int work_done;
3743
3744         priv->xstats.napi_poll++;
3745
3746         work_done = stmmac_tx_clean(priv, DMA_TX_SIZE, chan);
3747         work_done = min(work_done, budget);
3748
3749         if (work_done < budget)
3750                 napi_complete_done(napi, work_done);
3751
3752         /* Force transmission restart */
3753         tx_q = &priv->tx_queue[chan];
3754         if (tx_q->cur_tx != tx_q->dirty_tx) {
3755                 stmmac_enable_dma_transmission(priv, priv->ioaddr);
3756                 stmmac_set_tx_tail_ptr(priv, priv->ioaddr, tx_q->tx_tail_addr,
3757                                        chan);
3758         }
3759
3760         return work_done;
3761 }
3762
3763 /**
3764  *  stmmac_tx_timeout
3765  *  @dev : Pointer to net device structure
3766  *  Description: this function is called when a packet transmission fails to
3767  *   complete within a reasonable time. The driver will mark the error in the
3768  *   netdev structure and arrange for the device to be reset to a sane state
3769  *   in order to transmit a new packet.
3770  */
3771 static void stmmac_tx_timeout(struct net_device *dev)
3772 {
3773         struct stmmac_priv *priv = netdev_priv(dev);
3774
3775         stmmac_global_err(priv);
3776 }
3777
3778 /**
3779  *  stmmac_set_rx_mode - entry point for multicast addressing
3780  *  @dev : pointer to the device structure
3781  *  Description:
3782  *  This function is a driver entry point which gets called by the kernel
3783  *  whenever multicast addresses must be enabled/disabled.
3784  *  Return value:
3785  *  void.
3786  */
3787 static void stmmac_set_rx_mode(struct net_device *dev)
3788 {
3789         struct stmmac_priv *priv = netdev_priv(dev);
3790
3791         stmmac_set_filter(priv, priv->hw, dev);
3792 }
3793
3794 /**
3795  *  stmmac_change_mtu - entry point to change MTU size for the device.
3796  *  @dev : device pointer.
3797  *  @new_mtu : the new MTU size for the device.
3798  *  Description: the Maximum Transfer Unit (MTU) is used by the network layer
3799  *  to drive packet transmission. Ethernet has an MTU of 1500 octets
3800  *  (ETH_DATA_LEN). This value can be changed with ifconfig.
3801  *  Return value:
3802  *  0 on success and an appropriate (-)ve integer as defined in errno.h
3803  *  file on failure.
3804  */
3805 static int stmmac_change_mtu(struct net_device *dev, int new_mtu)
3806 {
3807         struct stmmac_priv *priv = netdev_priv(dev);
3808
3809         if (netif_running(dev)) {
3810                 netdev_err(priv->dev, "must be stopped to change its MTU\n");
3811                 return -EBUSY;
3812         }
3813
3814         dev->mtu = new_mtu;
3815
3816         netdev_update_features(dev);
3817
3818         return 0;
3819 }
3820
3821 static netdev_features_t stmmac_fix_features(struct net_device *dev,
3822                                              netdev_features_t features)
3823 {
3824         struct stmmac_priv *priv = netdev_priv(dev);
3825
3826         if (priv->plat->rx_coe == STMMAC_RX_COE_NONE)
3827                 features &= ~NETIF_F_RXCSUM;
3828
3829         if (!priv->plat->tx_coe)
3830                 features &= ~NETIF_F_CSUM_MASK;
3831
3832         /* Some GMAC devices have a bugged Jumbo frame support that
3833          * needs to have the Tx COE disabled for oversized frames
3834          * (due to limited buffer sizes). In this case we disable
3835          * the TX csum insertion in the TDES and not use SF.
3836          */
3837         if (priv->plat->bugged_jumbo && (dev->mtu > ETH_DATA_LEN))
3838                 features &= ~NETIF_F_CSUM_MASK;
3839
3840         /* Disable tso if asked by ethtool */
3841         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
3842                 if (features & NETIF_F_TSO)
3843                         priv->tso = true;
3844                 else
3845                         priv->tso = false;
3846         }
3847
3848         return features;
3849 }
3850
3851 static int stmmac_set_features(struct net_device *netdev,
3852                                netdev_features_t features)
3853 {
3854         struct stmmac_priv *priv = netdev_priv(netdev);
3855         bool sph_en;
3856         u32 chan;
3857
3858         /* Keep the COE Type in case of csum is supporting */
3859         if (features & NETIF_F_RXCSUM)
3860                 priv->hw->rx_csum = priv->plat->rx_coe;
3861         else
3862                 priv->hw->rx_csum = 0;
3863         /* No check needed because rx_coe has been set before and it will be
3864          * fixed in case of issue.
3865          */
3866         stmmac_rx_ipc(priv, priv->hw);
3867
3868         sph_en = (priv->hw->rx_csum > 0) && priv->sph;
3869         for (chan = 0; chan < priv->plat->rx_queues_to_use; chan++)
3870                 stmmac_enable_sph(priv, priv->ioaddr, sph_en, chan);
3871
3872         return 0;
3873 }
3874
3875 /**
3876  *  stmmac_interrupt - main ISR
3877  *  @irq: interrupt number.
3878  *  @dev_id: to pass the net device pointer.
3879  *  Description: this is the main driver interrupt service routine.
3880  *  It can call:
3881  *  o DMA service routine (to manage incoming frame reception and transmission
3882  *    status)
3883  *  o Core interrupts to manage: remote wake-up, management counter, LPI
3884  *    interrupts.
3885  */
3886 static irqreturn_t stmmac_interrupt(int irq, void *dev_id)
3887 {
3888         struct net_device *dev = (struct net_device *)dev_id;
3889         struct stmmac_priv *priv = netdev_priv(dev);
3890         u32 rx_cnt = priv->plat->rx_queues_to_use;
3891         u32 tx_cnt = priv->plat->tx_queues_to_use;
3892         u32 queues_count;
3893         u32 queue;
3894         bool xmac;
3895
3896         xmac = priv->plat->has_gmac4 || priv->plat->has_xgmac;
3897         queues_count = (rx_cnt > tx_cnt) ? rx_cnt : tx_cnt;
3898
3899         if (priv->irq_wake)
3900                 pm_wakeup_event(priv->device, 0);
3901
3902         if (unlikely(!dev)) {
3903                 netdev_err(priv->dev, "%s: invalid dev pointer\n", __func__);
3904                 return IRQ_NONE;
3905         }
3906
3907         /* Check if adapter is up */
3908         if (test_bit(STMMAC_DOWN, &priv->state))
3909                 return IRQ_HANDLED;
3910         /* Check if a fatal error happened */
3911         if (stmmac_safety_feat_interrupt(priv))
3912                 return IRQ_HANDLED;
3913
3914         /* To handle GMAC own interrupts */
3915         if ((priv->plat->has_gmac) || xmac) {
3916                 int status = stmmac_host_irq_status(priv, priv->hw, &priv->xstats);
3917                 int mtl_status;
3918
3919                 if (unlikely(status)) {
3920                         /* For LPI we need to save the tx status */
3921                         if (status & CORE_IRQ_TX_PATH_IN_LPI_MODE)
3922                                 priv->tx_path_in_lpi_mode = true;
3923                         if (status & CORE_IRQ_TX_PATH_EXIT_LPI_MODE)
3924                                 priv->tx_path_in_lpi_mode = false;
3925                 }
3926
3927                 for (queue = 0; queue < queues_count; queue++) {
3928                         struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
3929
3930                         mtl_status = stmmac_host_mtl_irq_status(priv, priv->hw,
3931                                                                 queue);
3932                         if (mtl_status != -EINVAL)
3933                                 status |= mtl_status;
3934
3935                         if (status & CORE_IRQ_MTL_RX_OVERFLOW)
3936                                 stmmac_set_rx_tail_ptr(priv, priv->ioaddr,
3937                                                        rx_q->rx_tail_addr,
3938                                                        queue);
3939                 }
3940
3941                 /* PCS link status */
3942                 if (priv->hw->pcs) {
3943                         if (priv->xstats.pcs_link)
3944                                 netif_carrier_on(dev);
3945                         else
3946                                 netif_carrier_off(dev);
3947                 }
3948         }
3949
3950         /* To handle DMA interrupts */
3951         stmmac_dma_interrupt(priv);
3952
3953         return IRQ_HANDLED;
3954 }
3955
3956 #ifdef CONFIG_NET_POLL_CONTROLLER
3957 /* Polling receive - used by NETCONSOLE and other diagnostic tools
3958  * to allow network I/O with interrupts disabled.
3959  */
3960 static void stmmac_poll_controller(struct net_device *dev)
3961 {
3962         disable_irq(dev->irq);
3963         stmmac_interrupt(dev->irq, dev);
3964         enable_irq(dev->irq);
3965 }
3966 #endif
3967
3968 /**
3969  *  stmmac_ioctl - Entry point for the Ioctl
3970  *  @dev: Device pointer.
3971  *  @rq: An IOCTL specefic structure, that can contain a pointer to
3972  *  a proprietary structure used to pass information to the driver.
3973  *  @cmd: IOCTL command
3974  *  Description:
3975  *  Currently it supports the phy_mii_ioctl(...) and HW time stamping.
3976  */
3977 static int stmmac_ioctl(struct net_device *dev, struct ifreq *rq, int cmd)
3978 {
3979         struct stmmac_priv *priv = netdev_priv (dev);
3980         int ret = -EOPNOTSUPP;
3981
3982         if (!netif_running(dev))
3983                 return -EINVAL;
3984
3985         switch (cmd) {
3986         case SIOCGMIIPHY:
3987         case SIOCGMIIREG:
3988         case SIOCSMIIREG:
3989                 ret = phylink_mii_ioctl(priv->phylink, rq, cmd);
3990                 break;
3991         case SIOCSHWTSTAMP:
3992                 ret = stmmac_hwtstamp_set(dev, rq);
3993                 break;
3994         case SIOCGHWTSTAMP:
3995                 ret = stmmac_hwtstamp_get(dev, rq);
3996                 break;
3997         default:
3998                 break;
3999         }
4000
4001         return ret;
4002 }
4003
4004 static int stmmac_setup_tc_block_cb(enum tc_setup_type type, void *type_data,
4005                                     void *cb_priv)
4006 {
4007         struct stmmac_priv *priv = cb_priv;
4008         int ret = -EOPNOTSUPP;
4009
4010         if (!tc_cls_can_offload_and_chain0(priv->dev, type_data))
4011                 return ret;
4012
4013         stmmac_disable_all_queues(priv);
4014
4015         switch (type) {
4016         case TC_SETUP_CLSU32:
4017                 ret = stmmac_tc_setup_cls_u32(priv, priv, type_data);
4018                 break;
4019         case TC_SETUP_CLSFLOWER:
4020                 ret = stmmac_tc_setup_cls(priv, priv, type_data);
4021                 break;
4022         default:
4023                 break;
4024         }
4025
4026         stmmac_enable_all_queues(priv);
4027         return ret;
4028 }
4029
4030 static LIST_HEAD(stmmac_block_cb_list);
4031
4032 static int stmmac_setup_tc(struct net_device *ndev, enum tc_setup_type type,
4033                            void *type_data)
4034 {
4035         struct stmmac_priv *priv = netdev_priv(ndev);
4036
4037         switch (type) {
4038         case TC_SETUP_BLOCK:
4039                 return flow_block_cb_setup_simple(type_data,
4040                                                   &stmmac_block_cb_list,
4041                                                   stmmac_setup_tc_block_cb,
4042                                                   priv, priv, true);
4043         case TC_SETUP_QDISC_CBS:
4044                 return stmmac_tc_setup_cbs(priv, priv, type_data);
4045         default:
4046                 return -EOPNOTSUPP;
4047         }
4048 }
4049
4050 static u16 stmmac_select_queue(struct net_device *dev, struct sk_buff *skb,
4051                                struct net_device *sb_dev)
4052 {
4053         int gso = skb_shinfo(skb)->gso_type;
4054
4055         if (gso & (SKB_GSO_TCPV4 | SKB_GSO_TCPV6 | SKB_GSO_UDP_L4)) {
4056                 /*
4057                  * There is no way to determine the number of TSO/USO
4058                  * capable Queues. Let's use always the Queue 0
4059                  * because if TSO/USO is supported then at least this
4060                  * one will be capable.
4061                  */
4062                 return 0;
4063         }
4064
4065         return netdev_pick_tx(dev, skb, NULL) % dev->real_num_tx_queues;
4066 }
4067
4068 static int stmmac_set_mac_address(struct net_device *ndev, void *addr)
4069 {
4070         struct stmmac_priv *priv = netdev_priv(ndev);
4071         int ret = 0;
4072
4073         ret = eth_mac_addr(ndev, addr);
4074         if (ret)
4075                 return ret;
4076
4077         stmmac_set_umac_addr(priv, priv->hw, ndev->dev_addr, 0);
4078
4079         return ret;
4080 }
4081
4082 #ifdef CONFIG_DEBUG_FS
4083 static struct dentry *stmmac_fs_dir;
4084
4085 static void sysfs_display_ring(void *head, int size, int extend_desc,
4086                                struct seq_file *seq)
4087 {
4088         int i;
4089         struct dma_extended_desc *ep = (struct dma_extended_desc *)head;
4090         struct dma_desc *p = (struct dma_desc *)head;
4091
4092         for (i = 0; i < size; i++) {
4093                 if (extend_desc) {
4094                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4095                                    i, (unsigned int)virt_to_phys(ep),
4096                                    le32_to_cpu(ep->basic.des0),
4097                                    le32_to_cpu(ep->basic.des1),
4098                                    le32_to_cpu(ep->basic.des2),
4099                                    le32_to_cpu(ep->basic.des3));
4100                         ep++;
4101                 } else {
4102                         seq_printf(seq, "%d [0x%x]: 0x%x 0x%x 0x%x 0x%x\n",
4103                                    i, (unsigned int)virt_to_phys(p),
4104                                    le32_to_cpu(p->des0), le32_to_cpu(p->des1),
4105                                    le32_to_cpu(p->des2), le32_to_cpu(p->des3));
4106                         p++;
4107                 }
4108                 seq_printf(seq, "\n");
4109         }
4110 }
4111
4112 static int stmmac_rings_status_show(struct seq_file *seq, void *v)
4113 {
4114         struct net_device *dev = seq->private;
4115         struct stmmac_priv *priv = netdev_priv(dev);
4116         u32 rx_count = priv->plat->rx_queues_to_use;
4117         u32 tx_count = priv->plat->tx_queues_to_use;
4118         u32 queue;
4119
4120         if ((dev->flags & IFF_UP) == 0)
4121                 return 0;
4122
4123         for (queue = 0; queue < rx_count; queue++) {
4124                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4125
4126                 seq_printf(seq, "RX Queue %d:\n", queue);
4127
4128                 if (priv->extend_desc) {
4129                         seq_printf(seq, "Extended descriptor ring:\n");
4130                         sysfs_display_ring((void *)rx_q->dma_erx,
4131                                            DMA_RX_SIZE, 1, seq);
4132                 } else {
4133                         seq_printf(seq, "Descriptor ring:\n");
4134                         sysfs_display_ring((void *)rx_q->dma_rx,
4135                                            DMA_RX_SIZE, 0, seq);
4136                 }
4137         }
4138
4139         for (queue = 0; queue < tx_count; queue++) {
4140                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4141
4142                 seq_printf(seq, "TX Queue %d:\n", queue);
4143
4144                 if (priv->extend_desc) {
4145                         seq_printf(seq, "Extended descriptor ring:\n");
4146                         sysfs_display_ring((void *)tx_q->dma_etx,
4147                                            DMA_TX_SIZE, 1, seq);
4148                 } else {
4149                         seq_printf(seq, "Descriptor ring:\n");
4150                         sysfs_display_ring((void *)tx_q->dma_tx,
4151                                            DMA_TX_SIZE, 0, seq);
4152                 }
4153         }
4154
4155         return 0;
4156 }
4157 DEFINE_SHOW_ATTRIBUTE(stmmac_rings_status);
4158
4159 static int stmmac_dma_cap_show(struct seq_file *seq, void *v)
4160 {
4161         struct net_device *dev = seq->private;
4162         struct stmmac_priv *priv = netdev_priv(dev);
4163
4164         if (!priv->hw_cap_support) {
4165                 seq_printf(seq, "DMA HW features not supported\n");
4166                 return 0;
4167         }
4168
4169         seq_printf(seq, "==============================\n");
4170         seq_printf(seq, "\tDMA HW features\n");
4171         seq_printf(seq, "==============================\n");
4172
4173         seq_printf(seq, "\t10/100 Mbps: %s\n",
4174                    (priv->dma_cap.mbps_10_100) ? "Y" : "N");
4175         seq_printf(seq, "\t1000 Mbps: %s\n",
4176                    (priv->dma_cap.mbps_1000) ? "Y" : "N");
4177         seq_printf(seq, "\tHalf duplex: %s\n",
4178                    (priv->dma_cap.half_duplex) ? "Y" : "N");
4179         seq_printf(seq, "\tHash Filter: %s\n",
4180                    (priv->dma_cap.hash_filter) ? "Y" : "N");
4181         seq_printf(seq, "\tMultiple MAC address registers: %s\n",
4182                    (priv->dma_cap.multi_addr) ? "Y" : "N");
4183         seq_printf(seq, "\tPCS (TBI/SGMII/RTBI PHY interfaces): %s\n",
4184                    (priv->dma_cap.pcs) ? "Y" : "N");
4185         seq_printf(seq, "\tSMA (MDIO) Interface: %s\n",
4186                    (priv->dma_cap.sma_mdio) ? "Y" : "N");
4187         seq_printf(seq, "\tPMT Remote wake up: %s\n",
4188                    (priv->dma_cap.pmt_remote_wake_up) ? "Y" : "N");
4189         seq_printf(seq, "\tPMT Magic Frame: %s\n",
4190                    (priv->dma_cap.pmt_magic_frame) ? "Y" : "N");
4191         seq_printf(seq, "\tRMON module: %s\n",
4192                    (priv->dma_cap.rmon) ? "Y" : "N");
4193         seq_printf(seq, "\tIEEE 1588-2002 Time Stamp: %s\n",
4194                    (priv->dma_cap.time_stamp) ? "Y" : "N");
4195         seq_printf(seq, "\tIEEE 1588-2008 Advanced Time Stamp: %s\n",
4196                    (priv->dma_cap.atime_stamp) ? "Y" : "N");
4197         seq_printf(seq, "\t802.3az - Energy-Efficient Ethernet (EEE): %s\n",
4198                    (priv->dma_cap.eee) ? "Y" : "N");
4199         seq_printf(seq, "\tAV features: %s\n", (priv->dma_cap.av) ? "Y" : "N");
4200         seq_printf(seq, "\tChecksum Offload in TX: %s\n",
4201                    (priv->dma_cap.tx_coe) ? "Y" : "N");
4202         if (priv->synopsys_id >= DWMAC_CORE_4_00) {
4203                 seq_printf(seq, "\tIP Checksum Offload in RX: %s\n",
4204                            (priv->dma_cap.rx_coe) ? "Y" : "N");
4205         } else {
4206                 seq_printf(seq, "\tIP Checksum Offload (type1) in RX: %s\n",
4207                            (priv->dma_cap.rx_coe_type1) ? "Y" : "N");
4208                 seq_printf(seq, "\tIP Checksum Offload (type2) in RX: %s\n",
4209                            (priv->dma_cap.rx_coe_type2) ? "Y" : "N");
4210         }
4211         seq_printf(seq, "\tRXFIFO > 2048bytes: %s\n",
4212                    (priv->dma_cap.rxfifo_over_2048) ? "Y" : "N");
4213         seq_printf(seq, "\tNumber of Additional RX channel: %d\n",
4214                    priv->dma_cap.number_rx_channel);
4215         seq_printf(seq, "\tNumber of Additional TX channel: %d\n",
4216                    priv->dma_cap.number_tx_channel);
4217         seq_printf(seq, "\tEnhanced descriptors: %s\n",
4218                    (priv->dma_cap.enh_desc) ? "Y" : "N");
4219
4220         return 0;
4221 }
4222 DEFINE_SHOW_ATTRIBUTE(stmmac_dma_cap);
4223
4224 static void stmmac_init_fs(struct net_device *dev)
4225 {
4226         struct stmmac_priv *priv = netdev_priv(dev);
4227
4228         /* Create per netdev entries */
4229         priv->dbgfs_dir = debugfs_create_dir(dev->name, stmmac_fs_dir);
4230
4231         /* Entry to report DMA RX/TX rings */
4232         debugfs_create_file("descriptors_status", 0444, priv->dbgfs_dir, dev,
4233                             &stmmac_rings_status_fops);
4234
4235         /* Entry to report the DMA HW features */
4236         debugfs_create_file("dma_cap", 0444, priv->dbgfs_dir, dev,
4237                             &stmmac_dma_cap_fops);
4238 }
4239
4240 static void stmmac_exit_fs(struct net_device *dev)
4241 {
4242         struct stmmac_priv *priv = netdev_priv(dev);
4243
4244         debugfs_remove_recursive(priv->dbgfs_dir);
4245 }
4246 #endif /* CONFIG_DEBUG_FS */
4247
4248 static u32 stmmac_vid_crc32_le(__le16 vid_le)
4249 {
4250         unsigned char *data = (unsigned char *)&vid_le;
4251         unsigned char data_byte = 0;
4252         u32 crc = ~0x0;
4253         u32 temp = 0;
4254         int i, bits;
4255
4256         bits = get_bitmask_order(VLAN_VID_MASK);
4257         for (i = 0; i < bits; i++) {
4258                 if ((i % 8) == 0)
4259                         data_byte = data[i / 8];
4260
4261                 temp = ((crc & 1) ^ data_byte) & 1;
4262                 crc >>= 1;
4263                 data_byte >>= 1;
4264
4265                 if (temp)
4266                         crc ^= 0xedb88320;
4267         }
4268
4269         return crc;
4270 }
4271
4272 static int stmmac_vlan_update(struct stmmac_priv *priv, bool is_double)
4273 {
4274         u32 crc, hash = 0;
4275         __le16 pmatch = 0;
4276         int count = 0;
4277         u16 vid = 0;
4278
4279         for_each_set_bit(vid, priv->active_vlans, VLAN_N_VID) {
4280                 __le16 vid_le = cpu_to_le16(vid);
4281                 crc = bitrev32(~stmmac_vid_crc32_le(vid_le)) >> 28;
4282                 hash |= (1 << crc);
4283                 count++;
4284         }
4285
4286         if (!priv->dma_cap.vlhash) {
4287                 if (count > 2) /* VID = 0 always passes filter */
4288                         return -EOPNOTSUPP;
4289
4290                 pmatch = cpu_to_le16(vid);
4291                 hash = 0;
4292         }
4293
4294         return stmmac_update_vlan_hash(priv, priv->hw, hash, pmatch, is_double);
4295 }
4296
4297 static int stmmac_vlan_rx_add_vid(struct net_device *ndev, __be16 proto, u16 vid)
4298 {
4299         struct stmmac_priv *priv = netdev_priv(ndev);
4300         bool is_double = false;
4301         int ret;
4302
4303         if (be16_to_cpu(proto) == ETH_P_8021AD)
4304                 is_double = true;
4305
4306         set_bit(vid, priv->active_vlans);
4307         ret = stmmac_vlan_update(priv, is_double);
4308         if (ret) {
4309                 clear_bit(vid, priv->active_vlans);
4310                 return ret;
4311         }
4312
4313         return ret;
4314 }
4315
4316 static int stmmac_vlan_rx_kill_vid(struct net_device *ndev, __be16 proto, u16 vid)
4317 {
4318         struct stmmac_priv *priv = netdev_priv(ndev);
4319         bool is_double = false;
4320
4321         if (be16_to_cpu(proto) == ETH_P_8021AD)
4322                 is_double = true;
4323
4324         clear_bit(vid, priv->active_vlans);
4325         return stmmac_vlan_update(priv, is_double);
4326 }
4327
4328 static const struct net_device_ops stmmac_netdev_ops = {
4329         .ndo_open = stmmac_open,
4330         .ndo_start_xmit = stmmac_xmit,
4331         .ndo_stop = stmmac_release,
4332         .ndo_change_mtu = stmmac_change_mtu,
4333         .ndo_fix_features = stmmac_fix_features,
4334         .ndo_set_features = stmmac_set_features,
4335         .ndo_set_rx_mode = stmmac_set_rx_mode,
4336         .ndo_tx_timeout = stmmac_tx_timeout,
4337         .ndo_do_ioctl = stmmac_ioctl,
4338         .ndo_setup_tc = stmmac_setup_tc,
4339         .ndo_select_queue = stmmac_select_queue,
4340 #ifdef CONFIG_NET_POLL_CONTROLLER
4341         .ndo_poll_controller = stmmac_poll_controller,
4342 #endif
4343         .ndo_set_mac_address = stmmac_set_mac_address,
4344         .ndo_vlan_rx_add_vid = stmmac_vlan_rx_add_vid,
4345         .ndo_vlan_rx_kill_vid = stmmac_vlan_rx_kill_vid,
4346 };
4347
4348 static void stmmac_reset_subtask(struct stmmac_priv *priv)
4349 {
4350         if (!test_and_clear_bit(STMMAC_RESET_REQUESTED, &priv->state))
4351                 return;
4352         if (test_bit(STMMAC_DOWN, &priv->state))
4353                 return;
4354
4355         netdev_err(priv->dev, "Reset adapter.\n");
4356
4357         rtnl_lock();
4358         netif_trans_update(priv->dev);
4359         while (test_and_set_bit(STMMAC_RESETING, &priv->state))
4360                 usleep_range(1000, 2000);
4361
4362         set_bit(STMMAC_DOWN, &priv->state);
4363         dev_close(priv->dev);
4364         dev_open(priv->dev, NULL);
4365         clear_bit(STMMAC_DOWN, &priv->state);
4366         clear_bit(STMMAC_RESETING, &priv->state);
4367         rtnl_unlock();
4368 }
4369
4370 static void stmmac_service_task(struct work_struct *work)
4371 {
4372         struct stmmac_priv *priv = container_of(work, struct stmmac_priv,
4373                         service_task);
4374
4375         stmmac_reset_subtask(priv);
4376         clear_bit(STMMAC_SERVICE_SCHED, &priv->state);
4377 }
4378
4379 /**
4380  *  stmmac_hw_init - Init the MAC device
4381  *  @priv: driver private structure
4382  *  Description: this function is to configure the MAC device according to
4383  *  some platform parameters or the HW capability register. It prepares the
4384  *  driver to use either ring or chain modes and to setup either enhanced or
4385  *  normal descriptors.
4386  */
4387 static int stmmac_hw_init(struct stmmac_priv *priv)
4388 {
4389         int ret;
4390
4391         /* dwmac-sun8i only work in chain mode */
4392         if (priv->plat->has_sun8i)
4393                 chain_mode = 1;
4394         priv->chain_mode = chain_mode;
4395
4396         /* Initialize HW Interface */
4397         ret = stmmac_hwif_init(priv);
4398         if (ret)
4399                 return ret;
4400
4401         /* Get the HW capability (new GMAC newer than 3.50a) */
4402         priv->hw_cap_support = stmmac_get_hw_features(priv);
4403         if (priv->hw_cap_support) {
4404                 dev_info(priv->device, "DMA HW capability register supported\n");
4405
4406                 /* We can override some gmac/dma configuration fields: e.g.
4407                  * enh_desc, tx_coe (e.g. that are passed through the
4408                  * platform) with the values from the HW capability
4409                  * register (if supported).
4410                  */
4411                 priv->plat->enh_desc = priv->dma_cap.enh_desc;
4412                 priv->plat->pmt = priv->dma_cap.pmt_remote_wake_up;
4413                 priv->hw->pmt = priv->plat->pmt;
4414                 if (priv->dma_cap.hash_tb_sz) {
4415                         priv->hw->multicast_filter_bins =
4416                                         (BIT(priv->dma_cap.hash_tb_sz) << 5);
4417                         priv->hw->mcast_bits_log2 =
4418                                         ilog2(priv->hw->multicast_filter_bins);
4419                 }
4420
4421                 /* TXCOE doesn't work in thresh DMA mode */
4422                 if (priv->plat->force_thresh_dma_mode)
4423                         priv->plat->tx_coe = 0;
4424                 else
4425                         priv->plat->tx_coe = priv->dma_cap.tx_coe;
4426
4427                 /* In case of GMAC4 rx_coe is from HW cap register. */
4428                 priv->plat->rx_coe = priv->dma_cap.rx_coe;
4429
4430                 if (priv->dma_cap.rx_coe_type2)
4431                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE2;
4432                 else if (priv->dma_cap.rx_coe_type1)
4433                         priv->plat->rx_coe = STMMAC_RX_COE_TYPE1;
4434
4435         } else {
4436                 dev_info(priv->device, "No HW DMA feature register supported\n");
4437         }
4438
4439         if (priv->plat->rx_coe) {
4440                 priv->hw->rx_csum = priv->plat->rx_coe;
4441                 dev_info(priv->device, "RX Checksum Offload Engine supported\n");
4442                 if (priv->synopsys_id < DWMAC_CORE_4_00)
4443                         dev_info(priv->device, "COE Type %d\n", priv->hw->rx_csum);
4444         }
4445         if (priv->plat->tx_coe)
4446                 dev_info(priv->device, "TX Checksum insertion supported\n");
4447
4448         if (priv->plat->pmt) {
4449                 dev_info(priv->device, "Wake-Up On Lan supported\n");
4450                 device_set_wakeup_capable(priv->device, 1);
4451         }
4452
4453         if (priv->dma_cap.tsoen)
4454                 dev_info(priv->device, "TSO supported\n");
4455
4456         /* Run HW quirks, if any */
4457         if (priv->hwif_quirks) {
4458                 ret = priv->hwif_quirks(priv);
4459                 if (ret)
4460                         return ret;
4461         }
4462
4463         /* Rx Watchdog is available in the COREs newer than the 3.40.
4464          * In some case, for example on bugged HW this feature
4465          * has to be disable and this can be done by passing the
4466          * riwt_off field from the platform.
4467          */
4468         if (((priv->synopsys_id >= DWMAC_CORE_3_50) ||
4469             (priv->plat->has_xgmac)) && (!priv->plat->riwt_off)) {
4470                 priv->use_riwt = 1;
4471                 dev_info(priv->device,
4472                          "Enable RX Mitigation via HW Watchdog Timer\n");
4473         }
4474
4475         return 0;
4476 }
4477
4478 /**
4479  * stmmac_dvr_probe
4480  * @device: device pointer
4481  * @plat_dat: platform data pointer
4482  * @res: stmmac resource pointer
4483  * Description: this is the main probe function used to
4484  * call the alloc_etherdev, allocate the priv structure.
4485  * Return:
4486  * returns 0 on success, otherwise errno.
4487  */
4488 int stmmac_dvr_probe(struct device *device,
4489                      struct plat_stmmacenet_data *plat_dat,
4490                      struct stmmac_resources *res)
4491 {
4492         struct net_device *ndev = NULL;
4493         struct stmmac_priv *priv;
4494         u32 queue, rxq, maxq;
4495         int i, ret = 0;
4496
4497         ndev = devm_alloc_etherdev_mqs(device, sizeof(struct stmmac_priv),
4498                                        MTL_MAX_TX_QUEUES, MTL_MAX_RX_QUEUES);
4499         if (!ndev)
4500                 return -ENOMEM;
4501
4502         SET_NETDEV_DEV(ndev, device);
4503
4504         priv = netdev_priv(ndev);
4505         priv->device = device;
4506         priv->dev = ndev;
4507
4508         stmmac_set_ethtool_ops(ndev);
4509         priv->pause = pause;
4510         priv->plat = plat_dat;
4511         priv->ioaddr = res->addr;
4512         priv->dev->base_addr = (unsigned long)res->addr;
4513
4514         priv->dev->irq = res->irq;
4515         priv->wol_irq = res->wol_irq;
4516         priv->lpi_irq = res->lpi_irq;
4517
4518         if (!IS_ERR_OR_NULL(res->mac))
4519                 memcpy(priv->dev->dev_addr, res->mac, ETH_ALEN);
4520
4521         dev_set_drvdata(device, priv->dev);
4522
4523         /* Verify driver arguments */
4524         stmmac_verify_args();
4525
4526         /* Allocate workqueue */
4527         priv->wq = create_singlethread_workqueue("stmmac_wq");
4528         if (!priv->wq) {
4529                 dev_err(priv->device, "failed to create workqueue\n");
4530                 return -ENOMEM;
4531         }
4532
4533         INIT_WORK(&priv->service_task, stmmac_service_task);
4534
4535         /* Override with kernel parameters if supplied XXX CRS XXX
4536          * this needs to have multiple instances
4537          */
4538         if ((phyaddr >= 0) && (phyaddr <= 31))
4539                 priv->plat->phy_addr = phyaddr;
4540
4541         if (priv->plat->stmmac_rst) {
4542                 ret = reset_control_assert(priv->plat->stmmac_rst);
4543                 reset_control_deassert(priv->plat->stmmac_rst);
4544                 /* Some reset controllers have only reset callback instead of
4545                  * assert + deassert callbacks pair.
4546                  */
4547                 if (ret == -ENOTSUPP)
4548                         reset_control_reset(priv->plat->stmmac_rst);
4549         }
4550
4551         /* Init MAC and get the capabilities */
4552         ret = stmmac_hw_init(priv);
4553         if (ret)
4554                 goto error_hw_init;
4555
4556         stmmac_check_ether_addr(priv);
4557
4558         /* Configure real RX and TX queues */
4559         netif_set_real_num_rx_queues(ndev, priv->plat->rx_queues_to_use);
4560         netif_set_real_num_tx_queues(ndev, priv->plat->tx_queues_to_use);
4561
4562         ndev->netdev_ops = &stmmac_netdev_ops;
4563
4564         ndev->hw_features = NETIF_F_SG | NETIF_F_IP_CSUM | NETIF_F_IPV6_CSUM |
4565                             NETIF_F_RXCSUM;
4566
4567         ret = stmmac_tc_init(priv, priv);
4568         if (!ret) {
4569                 ndev->hw_features |= NETIF_F_HW_TC;
4570         }
4571
4572         if ((priv->plat->tso_en) && (priv->dma_cap.tsoen)) {
4573                 ndev->hw_features |= NETIF_F_TSO | NETIF_F_TSO6;
4574                 if (priv->plat->has_gmac4)
4575                         ndev->hw_features |= NETIF_F_GSO_UDP_L4;
4576                 priv->tso = true;
4577                 dev_info(priv->device, "TSO feature enabled\n");
4578         }
4579
4580         if (priv->dma_cap.sphen) {
4581                 ndev->hw_features |= NETIF_F_GRO;
4582                 priv->sph = true;
4583                 dev_info(priv->device, "SPH feature enabled\n");
4584         }
4585
4586         if (priv->dma_cap.addr64) {
4587                 ret = dma_set_mask_and_coherent(device,
4588                                 DMA_BIT_MASK(priv->dma_cap.addr64));
4589                 if (!ret) {
4590                         dev_info(priv->device, "Using %d bits DMA width\n",
4591                                  priv->dma_cap.addr64);
4592
4593                         /*
4594                          * If more than 32 bits can be addressed, make sure to
4595                          * enable enhanced addressing mode.
4596                          */
4597                         if (IS_ENABLED(CONFIG_ARCH_DMA_ADDR_T_64BIT))
4598                                 priv->plat->dma_cfg->eame = true;
4599                 } else {
4600                         ret = dma_set_mask_and_coherent(device, DMA_BIT_MASK(32));
4601                         if (ret) {
4602                                 dev_err(priv->device, "Failed to set DMA Mask\n");
4603                                 goto error_hw_init;
4604                         }
4605
4606                         priv->dma_cap.addr64 = 32;
4607                 }
4608         }
4609
4610         ndev->features |= ndev->hw_features | NETIF_F_HIGHDMA;
4611         ndev->watchdog_timeo = msecs_to_jiffies(watchdog);
4612 #ifdef STMMAC_VLAN_TAG_USED
4613         /* Both mac100 and gmac support receive VLAN tag detection */
4614         ndev->features |= NETIF_F_HW_VLAN_CTAG_RX | NETIF_F_HW_VLAN_STAG_RX;
4615         if (priv->dma_cap.vlhash) {
4616                 ndev->features |= NETIF_F_HW_VLAN_CTAG_FILTER;
4617                 ndev->features |= NETIF_F_HW_VLAN_STAG_FILTER;
4618         }
4619         if (priv->dma_cap.vlins) {
4620                 ndev->features |= NETIF_F_HW_VLAN_CTAG_TX;
4621                 if (priv->dma_cap.dvlan)
4622                         ndev->features |= NETIF_F_HW_VLAN_STAG_TX;
4623         }
4624 #endif
4625         priv->msg_enable = netif_msg_init(debug, default_msg_level);
4626
4627         /* Initialize RSS */
4628         rxq = priv->plat->rx_queues_to_use;
4629         netdev_rss_key_fill(priv->rss.key, sizeof(priv->rss.key));
4630         for (i = 0; i < ARRAY_SIZE(priv->rss.table); i++)
4631                 priv->rss.table[i] = ethtool_rxfh_indir_default(i, rxq);
4632
4633         if (priv->dma_cap.rssen && priv->plat->rss_en)
4634                 ndev->features |= NETIF_F_RXHASH;
4635
4636         /* MTU range: 46 - hw-specific max */
4637         ndev->min_mtu = ETH_ZLEN - ETH_HLEN;
4638         if (priv->plat->has_xgmac)
4639                 ndev->max_mtu = XGMAC_JUMBO_LEN;
4640         else if ((priv->plat->enh_desc) || (priv->synopsys_id >= DWMAC_CORE_4_00))
4641                 ndev->max_mtu = JUMBO_LEN;
4642         else
4643                 ndev->max_mtu = SKB_MAX_HEAD(NET_SKB_PAD + NET_IP_ALIGN);
4644         /* Will not overwrite ndev->max_mtu if plat->maxmtu > ndev->max_mtu
4645          * as well as plat->maxmtu < ndev->min_mtu which is a invalid range.
4646          */
4647         if ((priv->plat->maxmtu < ndev->max_mtu) &&
4648             (priv->plat->maxmtu >= ndev->min_mtu))
4649                 ndev->max_mtu = priv->plat->maxmtu;
4650         else if (priv->plat->maxmtu < ndev->min_mtu)
4651                 dev_warn(priv->device,
4652                          "%s: warning: maxmtu having invalid value (%d)\n",
4653                          __func__, priv->plat->maxmtu);
4654
4655         if (flow_ctrl)
4656                 priv->flow_ctrl = FLOW_AUTO;    /* RX/TX pause on */
4657
4658         /* Setup channels NAPI */
4659         maxq = max(priv->plat->rx_queues_to_use, priv->plat->tx_queues_to_use);
4660
4661         for (queue = 0; queue < maxq; queue++) {
4662                 struct stmmac_channel *ch = &priv->channel[queue];
4663
4664                 ch->priv_data = priv;
4665                 ch->index = queue;
4666
4667                 if (queue < priv->plat->rx_queues_to_use) {
4668                         netif_napi_add(ndev, &ch->rx_napi, stmmac_napi_poll_rx,
4669                                        NAPI_POLL_WEIGHT);
4670                 }
4671                 if (queue < priv->plat->tx_queues_to_use) {
4672                         netif_tx_napi_add(ndev, &ch->tx_napi,
4673                                           stmmac_napi_poll_tx,
4674                                           NAPI_POLL_WEIGHT);
4675                 }
4676         }
4677
4678         mutex_init(&priv->lock);
4679
4680         /* If a specific clk_csr value is passed from the platform
4681          * this means that the CSR Clock Range selection cannot be
4682          * changed at run-time and it is fixed. Viceversa the driver'll try to
4683          * set the MDC clock dynamically according to the csr actual
4684          * clock input.
4685          */
4686         if (priv->plat->clk_csr >= 0)
4687                 priv->clk_csr = priv->plat->clk_csr;
4688         else
4689                 stmmac_clk_csr_set(priv);
4690
4691         stmmac_check_pcs_mode(priv);
4692
4693         if (priv->hw->pcs != STMMAC_PCS_RGMII  &&
4694             priv->hw->pcs != STMMAC_PCS_TBI &&
4695             priv->hw->pcs != STMMAC_PCS_RTBI) {
4696                 /* MDIO bus Registration */
4697                 ret = stmmac_mdio_register(ndev);
4698                 if (ret < 0) {
4699                         dev_err(priv->device,
4700                                 "%s: MDIO bus (id: %d) registration failed",
4701                                 __func__, priv->plat->bus_id);
4702                         goto error_mdio_register;
4703                 }
4704         }
4705
4706         ret = stmmac_phy_setup(priv);
4707         if (ret) {
4708                 netdev_err(ndev, "failed to setup phy (%d)\n", ret);
4709                 goto error_phy_setup;
4710         }
4711
4712         ret = register_netdev(ndev);
4713         if (ret) {
4714                 dev_err(priv->device, "%s: ERROR %i registering the device\n",
4715                         __func__, ret);
4716                 goto error_netdev_register;
4717         }
4718
4719 #ifdef CONFIG_DEBUG_FS
4720         stmmac_init_fs(ndev);
4721 #endif
4722
4723         return ret;
4724
4725 error_netdev_register:
4726         phylink_destroy(priv->phylink);
4727 error_phy_setup:
4728         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4729             priv->hw->pcs != STMMAC_PCS_TBI &&
4730             priv->hw->pcs != STMMAC_PCS_RTBI)
4731                 stmmac_mdio_unregister(ndev);
4732 error_mdio_register:
4733         for (queue = 0; queue < maxq; queue++) {
4734                 struct stmmac_channel *ch = &priv->channel[queue];
4735
4736                 if (queue < priv->plat->rx_queues_to_use)
4737                         netif_napi_del(&ch->rx_napi);
4738                 if (queue < priv->plat->tx_queues_to_use)
4739                         netif_napi_del(&ch->tx_napi);
4740         }
4741 error_hw_init:
4742         destroy_workqueue(priv->wq);
4743
4744         return ret;
4745 }
4746 EXPORT_SYMBOL_GPL(stmmac_dvr_probe);
4747
4748 /**
4749  * stmmac_dvr_remove
4750  * @dev: device pointer
4751  * Description: this function resets the TX/RX processes, disables the MAC RX/TX
4752  * changes the link status, releases the DMA descriptor rings.
4753  */
4754 int stmmac_dvr_remove(struct device *dev)
4755 {
4756         struct net_device *ndev = dev_get_drvdata(dev);
4757         struct stmmac_priv *priv = netdev_priv(ndev);
4758
4759         netdev_info(priv->dev, "%s: removing driver", __func__);
4760
4761 #ifdef CONFIG_DEBUG_FS
4762         stmmac_exit_fs(ndev);
4763 #endif
4764         stmmac_stop_all_dma(priv);
4765
4766         stmmac_mac_set(priv, priv->ioaddr, false);
4767         netif_carrier_off(ndev);
4768         unregister_netdev(ndev);
4769         phylink_destroy(priv->phylink);
4770         if (priv->plat->stmmac_rst)
4771                 reset_control_assert(priv->plat->stmmac_rst);
4772         clk_disable_unprepare(priv->plat->pclk);
4773         clk_disable_unprepare(priv->plat->stmmac_clk);
4774         if (priv->hw->pcs != STMMAC_PCS_RGMII &&
4775             priv->hw->pcs != STMMAC_PCS_TBI &&
4776             priv->hw->pcs != STMMAC_PCS_RTBI)
4777                 stmmac_mdio_unregister(ndev);
4778         destroy_workqueue(priv->wq);
4779         mutex_destroy(&priv->lock);
4780
4781         return 0;
4782 }
4783 EXPORT_SYMBOL_GPL(stmmac_dvr_remove);
4784
4785 /**
4786  * stmmac_suspend - suspend callback
4787  * @dev: device pointer
4788  * Description: this is the function to suspend the device and it is called
4789  * by the platform driver to stop the network queue, release the resources,
4790  * program the PMT register (for WoL), clean and release driver resources.
4791  */
4792 int stmmac_suspend(struct device *dev)
4793 {
4794         struct net_device *ndev = dev_get_drvdata(dev);
4795         struct stmmac_priv *priv = netdev_priv(ndev);
4796
4797         if (!ndev || !netif_running(ndev))
4798                 return 0;
4799
4800         phylink_mac_change(priv->phylink, false);
4801
4802         mutex_lock(&priv->lock);
4803
4804         netif_device_detach(ndev);
4805         stmmac_stop_all_queues(priv);
4806
4807         stmmac_disable_all_queues(priv);
4808
4809         /* Stop TX/RX DMA */
4810         stmmac_stop_all_dma(priv);
4811
4812         /* Enable Power down mode by programming the PMT regs */
4813         if (device_may_wakeup(priv->device)) {
4814                 stmmac_pmt(priv, priv->hw, priv->wolopts);
4815                 priv->irq_wake = 1;
4816         } else {
4817                 mutex_unlock(&priv->lock);
4818                 rtnl_lock();
4819                 phylink_stop(priv->phylink);
4820                 rtnl_unlock();
4821                 mutex_lock(&priv->lock);
4822
4823                 stmmac_mac_set(priv, priv->ioaddr, false);
4824                 pinctrl_pm_select_sleep_state(priv->device);
4825                 /* Disable clock in case of PWM is off */
4826                 if (priv->plat->clk_ptp_ref)
4827                         clk_disable_unprepare(priv->plat->clk_ptp_ref);
4828                 clk_disable_unprepare(priv->plat->pclk);
4829                 clk_disable_unprepare(priv->plat->stmmac_clk);
4830         }
4831         mutex_unlock(&priv->lock);
4832
4833         priv->speed = SPEED_UNKNOWN;
4834         return 0;
4835 }
4836 EXPORT_SYMBOL_GPL(stmmac_suspend);
4837
4838 /**
4839  * stmmac_reset_queues_param - reset queue parameters
4840  * @dev: device pointer
4841  */
4842 static void stmmac_reset_queues_param(struct stmmac_priv *priv)
4843 {
4844         u32 rx_cnt = priv->plat->rx_queues_to_use;
4845         u32 tx_cnt = priv->plat->tx_queues_to_use;
4846         u32 queue;
4847
4848         for (queue = 0; queue < rx_cnt; queue++) {
4849                 struct stmmac_rx_queue *rx_q = &priv->rx_queue[queue];
4850
4851                 rx_q->cur_rx = 0;
4852                 rx_q->dirty_rx = 0;
4853         }
4854
4855         for (queue = 0; queue < tx_cnt; queue++) {
4856                 struct stmmac_tx_queue *tx_q = &priv->tx_queue[queue];
4857
4858                 tx_q->cur_tx = 0;
4859                 tx_q->dirty_tx = 0;
4860                 tx_q->mss = 0;
4861         }
4862 }
4863
4864 /**
4865  * stmmac_resume - resume callback
4866  * @dev: device pointer
4867  * Description: when resume this function is invoked to setup the DMA and CORE
4868  * in a usable state.
4869  */
4870 int stmmac_resume(struct device *dev)
4871 {
4872         struct net_device *ndev = dev_get_drvdata(dev);
4873         struct stmmac_priv *priv = netdev_priv(ndev);
4874
4875         if (!netif_running(ndev))
4876                 return 0;
4877
4878         /* Power Down bit, into the PM register, is cleared
4879          * automatically as soon as a magic packet or a Wake-up frame
4880          * is received. Anyway, it's better to manually clear
4881          * this bit because it can generate problems while resuming
4882          * from another devices (e.g. serial console).
4883          */
4884         if (device_may_wakeup(priv->device)) {
4885                 mutex_lock(&priv->lock);
4886                 stmmac_pmt(priv, priv->hw, 0);
4887                 mutex_unlock(&priv->lock);
4888                 priv->irq_wake = 0;
4889         } else {
4890                 pinctrl_pm_select_default_state(priv->device);
4891                 /* enable the clk previously disabled */
4892                 clk_prepare_enable(priv->plat->stmmac_clk);
4893                 clk_prepare_enable(priv->plat->pclk);
4894                 if (priv->plat->clk_ptp_ref)
4895                         clk_prepare_enable(priv->plat->clk_ptp_ref);
4896                 /* reset the phy so that it's ready */
4897                 if (priv->mii)
4898                         stmmac_mdio_reset(priv->mii);
4899         }
4900
4901         netif_device_attach(ndev);
4902
4903         mutex_lock(&priv->lock);
4904
4905         stmmac_reset_queues_param(priv);
4906
4907         stmmac_clear_descriptors(priv);
4908
4909         stmmac_hw_setup(ndev, false);
4910         stmmac_init_coalesce(priv);
4911         stmmac_set_rx_mode(ndev);
4912
4913         stmmac_enable_all_queues(priv);
4914
4915         stmmac_start_all_queues(priv);
4916
4917         mutex_unlock(&priv->lock);
4918
4919         if (!device_may_wakeup(priv->device)) {
4920                 rtnl_lock();
4921                 phylink_start(priv->phylink);
4922                 rtnl_unlock();
4923         }
4924
4925         phylink_mac_change(priv->phylink, true);
4926
4927         return 0;
4928 }
4929 EXPORT_SYMBOL_GPL(stmmac_resume);
4930
4931 #ifndef MODULE
4932 static int __init stmmac_cmdline_opt(char *str)
4933 {
4934         char *opt;
4935
4936         if (!str || !*str)
4937                 return -EINVAL;
4938         while ((opt = strsep(&str, ",")) != NULL) {
4939                 if (!strncmp(opt, "debug:", 6)) {
4940                         if (kstrtoint(opt + 6, 0, &debug))
4941                                 goto err;
4942                 } else if (!strncmp(opt, "phyaddr:", 8)) {
4943                         if (kstrtoint(opt + 8, 0, &phyaddr))
4944                                 goto err;
4945                 } else if (!strncmp(opt, "buf_sz:", 7)) {
4946                         if (kstrtoint(opt + 7, 0, &buf_sz))
4947                                 goto err;
4948                 } else if (!strncmp(opt, "tc:", 3)) {
4949                         if (kstrtoint(opt + 3, 0, &tc))
4950                                 goto err;
4951                 } else if (!strncmp(opt, "watchdog:", 9)) {
4952                         if (kstrtoint(opt + 9, 0, &watchdog))
4953                                 goto err;
4954                 } else if (!strncmp(opt, "flow_ctrl:", 10)) {
4955                         if (kstrtoint(opt + 10, 0, &flow_ctrl))
4956                                 goto err;
4957                 } else if (!strncmp(opt, "pause:", 6)) {
4958                         if (kstrtoint(opt + 6, 0, &pause))
4959                                 goto err;
4960                 } else if (!strncmp(opt, "eee_timer:", 10)) {
4961                         if (kstrtoint(opt + 10, 0, &eee_timer))
4962                                 goto err;
4963                 } else if (!strncmp(opt, "chain_mode:", 11)) {
4964                         if (kstrtoint(opt + 11, 0, &chain_mode))
4965                                 goto err;
4966                 }
4967         }
4968         return 0;
4969
4970 err:
4971         pr_err("%s: ERROR broken module parameter conversion", __func__);
4972         return -EINVAL;
4973 }
4974
4975 __setup("stmmaceth=", stmmac_cmdline_opt);
4976 #endif /* MODULE */
4977
4978 static int __init stmmac_init(void)
4979 {
4980 #ifdef CONFIG_DEBUG_FS
4981         /* Create debugfs main directory if it doesn't exist yet */
4982         if (!stmmac_fs_dir)
4983                 stmmac_fs_dir = debugfs_create_dir(STMMAC_RESOURCE_NAME, NULL);
4984 #endif
4985
4986         return 0;
4987 }
4988
4989 static void __exit stmmac_exit(void)
4990 {
4991 #ifdef CONFIG_DEBUG_FS
4992         debugfs_remove_recursive(stmmac_fs_dir);
4993 #endif
4994 }
4995
4996 module_init(stmmac_init)
4997 module_exit(stmmac_exit)
4998
4999 MODULE_DESCRIPTION("STMMAC 10/100/1000 Ethernet device driver");
5000 MODULE_AUTHOR("Giuseppe Cavallaro <peppe.cavallaro@st.com>");
5001 MODULE_LICENSE("GPL");