1 // SPDX-License-Identifier: (GPL-2.0+ OR BSD-3-Clause)
2 /* Copyright 2017-2019 NXP */
7 #include <linux/of_mdio.h>
8 #include <linux/vmalloc.h>
10 /* ENETC overhead: optional extension BD + 1 BD gap */
11 #define ENETC_TXBDS_NEEDED(val) ((val) + 2)
12 /* max # of chained Tx BDs is 15, including head and extension BD */
13 #define ENETC_MAX_SKB_FRAGS 13
14 #define ENETC_TXBDS_MAX_NEEDED ENETC_TXBDS_NEEDED(ENETC_MAX_SKB_FRAGS + 1)
16 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
19 netdev_tx_t enetc_xmit(struct sk_buff *skb, struct net_device *ndev)
21 struct enetc_ndev_priv *priv = netdev_priv(ndev);
22 struct enetc_bdr *tx_ring;
25 tx_ring = priv->tx_ring[skb->queue_mapping];
27 if (unlikely(skb_shinfo(skb)->nr_frags > ENETC_MAX_SKB_FRAGS))
28 if (unlikely(skb_linearize(skb)))
31 count = skb_shinfo(skb)->nr_frags + 1; /* fragments + head */
32 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_NEEDED(count)) {
33 netif_stop_subqueue(ndev, tx_ring->index);
34 return NETDEV_TX_BUSY;
37 count = enetc_map_tx_buffs(tx_ring, skb, priv->active_offloads);
41 if (enetc_bd_unused(tx_ring) < ENETC_TXBDS_MAX_NEEDED)
42 netif_stop_subqueue(ndev, tx_ring->index);
47 dev_kfree_skb_any(skb);
51 static bool enetc_tx_csum(struct sk_buff *skb, union enetc_tx_bd *txbd)
53 int l3_start, l3_hsize;
54 u16 l3_flags, l4_flags;
56 if (skb->ip_summed != CHECKSUM_PARTIAL)
59 switch (skb->csum_offset) {
60 case offsetof(struct tcphdr, check):
61 l4_flags = ENETC_TXBD_L4_TCP;
63 case offsetof(struct udphdr, check):
64 l4_flags = ENETC_TXBD_L4_UDP;
67 skb_checksum_help(skb);
71 l3_start = skb_network_offset(skb);
72 l3_hsize = skb_network_header_len(skb);
75 if (skb->protocol == htons(ETH_P_IPV6))
76 l3_flags = ENETC_TXBD_L3_IPV6;
79 txbd->l3_csoff = enetc_txbd_l3_csoff(l3_start, l3_hsize, l3_flags);
80 txbd->l4_csoff = l4_flags;
85 static void enetc_unmap_tx_buff(struct enetc_bdr *tx_ring,
86 struct enetc_tx_swbd *tx_swbd)
88 if (tx_swbd->is_dma_page)
89 dma_unmap_page(tx_ring->dev, tx_swbd->dma,
90 tx_swbd->len, DMA_TO_DEVICE);
92 dma_unmap_single(tx_ring->dev, tx_swbd->dma,
93 tx_swbd->len, DMA_TO_DEVICE);
97 static void enetc_free_tx_skb(struct enetc_bdr *tx_ring,
98 struct enetc_tx_swbd *tx_swbd)
101 enetc_unmap_tx_buff(tx_ring, tx_swbd);
104 dev_kfree_skb_any(tx_swbd->skb);
109 static int enetc_map_tx_buffs(struct enetc_bdr *tx_ring, struct sk_buff *skb,
112 struct enetc_tx_swbd *tx_swbd;
114 int len = skb_headlen(skb);
115 union enetc_tx_bd temp_bd;
116 union enetc_tx_bd *txbd;
117 bool do_vlan, do_tstamp;
123 i = tx_ring->next_to_use;
124 txbd = ENETC_TXBD(*tx_ring, i);
127 dma = dma_map_single(tx_ring->dev, skb->data, len, DMA_TO_DEVICE);
128 if (unlikely(dma_mapping_error(tx_ring->dev, dma)))
131 temp_bd.addr = cpu_to_le64(dma);
132 temp_bd.buf_len = cpu_to_le16(len);
135 tx_swbd = &tx_ring->tx_swbd[i];
138 tx_swbd->is_dma_page = 0;
141 do_vlan = skb_vlan_tag_present(skb);
142 do_tstamp = (active_offloads & ENETC_F_TX_TSTAMP) &&
143 (skb_shinfo(skb)->tx_flags & SKBTX_HW_TSTAMP);
144 tx_swbd->do_tstamp = do_tstamp;
145 tx_swbd->check_wb = tx_swbd->do_tstamp;
147 if (do_vlan || do_tstamp)
148 flags |= ENETC_TXBD_FLAGS_EX;
150 if (enetc_tx_csum(skb, &temp_bd))
151 flags |= ENETC_TXBD_FLAGS_CSUM | ENETC_TXBD_FLAGS_L4CS;
153 /* first BD needs frm_len and offload flags set */
154 temp_bd.frm_len = cpu_to_le16(skb->len);
155 temp_bd.flags = flags;
157 if (flags & ENETC_TXBD_FLAGS_EX) {
160 enetc_clear_tx_bd(&temp_bd);
162 /* add extension BD for VLAN and/or timestamping */
167 if (unlikely(i == tx_ring->bd_count)) {
169 tx_swbd = tx_ring->tx_swbd;
170 txbd = ENETC_TXBD(*tx_ring, 0);
175 temp_bd.ext.vid = cpu_to_le16(skb_vlan_tag_get(skb));
176 temp_bd.ext.tpid = 0; /* < C-TAG */
177 e_flags |= ENETC_TXBD_E_FLAGS_VLAN_INS;
181 skb_shinfo(skb)->tx_flags |= SKBTX_IN_PROGRESS;
182 e_flags |= ENETC_TXBD_E_FLAGS_TWO_STEP_PTP;
185 temp_bd.ext.e_flags = e_flags;
189 frag = &skb_shinfo(skb)->frags[0];
190 for (f = 0; f < skb_shinfo(skb)->nr_frags; f++, frag++) {
191 len = skb_frag_size(frag);
192 dma = skb_frag_dma_map(tx_ring->dev, frag, 0, len,
194 if (dma_mapping_error(tx_ring->dev, dma))
198 enetc_clear_tx_bd(&temp_bd);
204 if (unlikely(i == tx_ring->bd_count)) {
206 tx_swbd = tx_ring->tx_swbd;
207 txbd = ENETC_TXBD(*tx_ring, 0);
211 temp_bd.addr = cpu_to_le64(dma);
212 temp_bd.buf_len = cpu_to_le16(len);
216 tx_swbd->is_dma_page = 1;
220 /* last BD needs 'F' bit set */
221 flags |= ENETC_TXBD_FLAGS_F;
222 temp_bd.flags = flags;
225 tx_ring->tx_swbd[i].skb = skb;
227 enetc_bdr_idx_inc(tx_ring, &i);
228 tx_ring->next_to_use = i;
230 /* let H/W know BD ring has been updated */
231 enetc_wr_reg(tx_ring->tpir, i); /* includes wmb() */
236 dev_err(tx_ring->dev, "DMA map error");
239 tx_swbd = &tx_ring->tx_swbd[i];
240 enetc_free_tx_skb(tx_ring, tx_swbd);
242 i = tx_ring->bd_count;
249 static irqreturn_t enetc_msix(int irq, void *data)
251 struct enetc_int_vector *v = data;
254 /* disable interrupts */
255 enetc_wr_reg(v->rbier, 0);
257 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings)
258 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i), 0);
260 napi_schedule_irqoff(&v->napi);
265 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget);
266 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
267 struct napi_struct *napi, int work_limit);
269 static int enetc_poll(struct napi_struct *napi, int budget)
271 struct enetc_int_vector
272 *v = container_of(napi, struct enetc_int_vector, napi);
273 bool complete = true;
277 for (i = 0; i < v->count_tx_rings; i++)
278 if (!enetc_clean_tx_ring(&v->tx_ring[i], budget))
281 work_done = enetc_clean_rx_ring(&v->rx_ring, napi, budget);
282 if (work_done == budget)
288 napi_complete_done(napi, work_done);
290 /* enable interrupts */
291 enetc_wr_reg(v->rbier, ENETC_RBIER_RXTIE);
293 for_each_set_bit(i, &v->tx_rings_map, v->count_tx_rings)
294 enetc_wr_reg(v->tbier_base + ENETC_BDR_OFF(i),
300 static int enetc_bd_ready_count(struct enetc_bdr *tx_ring, int ci)
302 int pi = enetc_rd_reg(tx_ring->tcir) & ENETC_TBCIR_IDX_MASK;
304 return pi >= ci ? pi - ci : tx_ring->bd_count - ci + pi;
307 static void enetc_get_tx_tstamp(struct enetc_hw *hw, union enetc_tx_bd *txbd,
310 u32 lo, hi, tstamp_lo;
312 lo = enetc_rd(hw, ENETC_SICTR0);
313 hi = enetc_rd(hw, ENETC_SICTR1);
314 tstamp_lo = le32_to_cpu(txbd->wb.tstamp);
317 *tstamp = (u64)hi << 32 | tstamp_lo;
320 static void enetc_tstamp_tx(struct sk_buff *skb, u64 tstamp)
322 struct skb_shared_hwtstamps shhwtstamps;
324 if (skb_shinfo(skb)->tx_flags & SKBTX_IN_PROGRESS) {
325 memset(&shhwtstamps, 0, sizeof(shhwtstamps));
326 shhwtstamps.hwtstamp = ns_to_ktime(tstamp);
327 skb_tstamp_tx(skb, &shhwtstamps);
331 static bool enetc_clean_tx_ring(struct enetc_bdr *tx_ring, int napi_budget)
333 struct net_device *ndev = tx_ring->ndev;
334 int tx_frm_cnt = 0, tx_byte_cnt = 0;
335 struct enetc_tx_swbd *tx_swbd;
340 i = tx_ring->next_to_clean;
341 tx_swbd = &tx_ring->tx_swbd[i];
342 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
346 while (bds_to_clean && tx_frm_cnt < ENETC_DEFAULT_TX_WORK) {
347 bool is_eof = !!tx_swbd->skb;
349 if (unlikely(tx_swbd->check_wb)) {
350 struct enetc_ndev_priv *priv = netdev_priv(ndev);
351 union enetc_tx_bd *txbd;
353 txbd = ENETC_TXBD(*tx_ring, i);
355 if (txbd->flags & ENETC_TXBD_FLAGS_W &&
356 tx_swbd->do_tstamp) {
357 enetc_get_tx_tstamp(&priv->si->hw, txbd,
363 if (likely(tx_swbd->dma))
364 enetc_unmap_tx_buff(tx_ring, tx_swbd);
367 if (unlikely(do_tstamp)) {
368 enetc_tstamp_tx(tx_swbd->skb, tstamp);
371 napi_consume_skb(tx_swbd->skb, napi_budget);
375 tx_byte_cnt += tx_swbd->len;
380 if (unlikely(i == tx_ring->bd_count)) {
382 tx_swbd = tx_ring->tx_swbd;
385 /* BD iteration loop end */
388 /* re-arm interrupt source */
389 enetc_wr_reg(tx_ring->idr, BIT(tx_ring->index) |
390 BIT(16 + tx_ring->index));
393 if (unlikely(!bds_to_clean))
394 bds_to_clean = enetc_bd_ready_count(tx_ring, i);
397 tx_ring->next_to_clean = i;
398 tx_ring->stats.packets += tx_frm_cnt;
399 tx_ring->stats.bytes += tx_byte_cnt;
401 if (unlikely(tx_frm_cnt && netif_carrier_ok(ndev) &&
402 __netif_subqueue_stopped(ndev, tx_ring->index) &&
403 (enetc_bd_unused(tx_ring) >= ENETC_TXBDS_MAX_NEEDED))) {
404 netif_wake_subqueue(ndev, tx_ring->index);
407 return tx_frm_cnt != ENETC_DEFAULT_TX_WORK;
410 static bool enetc_new_page(struct enetc_bdr *rx_ring,
411 struct enetc_rx_swbd *rx_swbd)
416 page = dev_alloc_page();
420 addr = dma_map_page(rx_ring->dev, page, 0, PAGE_SIZE, DMA_FROM_DEVICE);
421 if (unlikely(dma_mapping_error(rx_ring->dev, addr))) {
428 rx_swbd->page = page;
429 rx_swbd->page_offset = ENETC_RXB_PAD;
434 static int enetc_refill_rx_ring(struct enetc_bdr *rx_ring, const int buff_cnt)
436 struct enetc_rx_swbd *rx_swbd;
437 union enetc_rx_bd *rxbd;
440 i = rx_ring->next_to_use;
441 rx_swbd = &rx_ring->rx_swbd[i];
442 rxbd = ENETC_RXBD(*rx_ring, i);
444 for (j = 0; j < buff_cnt; j++) {
446 if (unlikely(!rx_swbd->page)) {
447 if (unlikely(!enetc_new_page(rx_ring, rx_swbd))) {
448 rx_ring->stats.rx_alloc_errs++;
454 rxbd->w.addr = cpu_to_le64(rx_swbd->dma +
455 rx_swbd->page_offset);
456 /* clear 'R" as well */
462 if (unlikely(i == rx_ring->bd_count)) {
464 rx_swbd = rx_ring->rx_swbd;
465 rxbd = ENETC_RXBD(*rx_ring, 0);
470 rx_ring->next_to_alloc = i; /* keep track from page reuse */
471 rx_ring->next_to_use = i;
472 /* update ENETC's consumer index */
473 enetc_wr_reg(rx_ring->rcir, i);
479 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
480 static void enetc_get_rx_tstamp(struct net_device *ndev,
481 union enetc_rx_bd *rxbd,
484 struct skb_shared_hwtstamps *shhwtstamps = skb_hwtstamps(skb);
485 struct enetc_ndev_priv *priv = netdev_priv(ndev);
486 struct enetc_hw *hw = &priv->si->hw;
487 u32 lo, hi, tstamp_lo;
490 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_TSTMP) {
491 lo = enetc_rd(hw, ENETC_SICTR0);
492 hi = enetc_rd(hw, ENETC_SICTR1);
493 tstamp_lo = le32_to_cpu(rxbd->r.tstamp);
497 tstamp = (u64)hi << 32 | tstamp_lo;
498 memset(shhwtstamps, 0, sizeof(*shhwtstamps));
499 shhwtstamps->hwtstamp = ns_to_ktime(tstamp);
504 static void enetc_get_offloads(struct enetc_bdr *rx_ring,
505 union enetc_rx_bd *rxbd, struct sk_buff *skb)
507 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
508 struct enetc_ndev_priv *priv = netdev_priv(rx_ring->ndev);
511 if (rx_ring->ndev->features & NETIF_F_RXCSUM) {
512 u16 inet_csum = le16_to_cpu(rxbd->r.inet_csum);
514 skb->csum = csum_unfold((__force __sum16)~htons(inet_csum));
515 skb->ip_summed = CHECKSUM_COMPLETE;
518 /* copy VLAN to skb, if one is extracted, for now we assume it's a
519 * standard TPID, but HW also supports custom values
521 if (le16_to_cpu(rxbd->r.flags) & ENETC_RXBD_FLAG_VLAN)
522 __vlan_hwaccel_put_tag(skb, htons(ETH_P_8021Q),
523 le16_to_cpu(rxbd->r.vlan_opt));
524 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
525 if (priv->active_offloads & ENETC_F_RX_TSTAMP)
526 enetc_get_rx_tstamp(rx_ring->ndev, rxbd, skb);
530 static void enetc_process_skb(struct enetc_bdr *rx_ring,
533 skb_record_rx_queue(skb, rx_ring->index);
534 skb->protocol = eth_type_trans(skb, rx_ring->ndev);
537 static bool enetc_page_reusable(struct page *page)
539 return (!page_is_pfmemalloc(page) && page_ref_count(page) == 1);
542 static void enetc_reuse_page(struct enetc_bdr *rx_ring,
543 struct enetc_rx_swbd *old)
545 struct enetc_rx_swbd *new;
547 new = &rx_ring->rx_swbd[rx_ring->next_to_alloc];
549 /* next buf that may reuse a page */
550 enetc_bdr_idx_inc(rx_ring, &rx_ring->next_to_alloc);
552 /* copy page reference */
556 static struct enetc_rx_swbd *enetc_get_rx_buff(struct enetc_bdr *rx_ring,
559 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
561 dma_sync_single_range_for_cpu(rx_ring->dev, rx_swbd->dma,
562 rx_swbd->page_offset,
563 size, DMA_FROM_DEVICE);
567 static void enetc_put_rx_buff(struct enetc_bdr *rx_ring,
568 struct enetc_rx_swbd *rx_swbd)
570 if (likely(enetc_page_reusable(rx_swbd->page))) {
571 rx_swbd->page_offset ^= ENETC_RXB_TRUESIZE;
572 page_ref_inc(rx_swbd->page);
574 enetc_reuse_page(rx_ring, rx_swbd);
576 /* sync for use by the device */
577 dma_sync_single_range_for_device(rx_ring->dev, rx_swbd->dma,
578 rx_swbd->page_offset,
582 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
583 PAGE_SIZE, DMA_FROM_DEVICE);
586 rx_swbd->page = NULL;
589 static struct sk_buff *enetc_map_rx_buff_to_skb(struct enetc_bdr *rx_ring,
592 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
596 ba = page_address(rx_swbd->page) + rx_swbd->page_offset;
597 skb = build_skb(ba - ENETC_RXB_PAD, ENETC_RXB_TRUESIZE);
598 if (unlikely(!skb)) {
599 rx_ring->stats.rx_alloc_errs++;
603 skb_reserve(skb, ENETC_RXB_PAD);
604 __skb_put(skb, size);
606 enetc_put_rx_buff(rx_ring, rx_swbd);
611 static void enetc_add_rx_buff_to_skb(struct enetc_bdr *rx_ring, int i,
612 u16 size, struct sk_buff *skb)
614 struct enetc_rx_swbd *rx_swbd = enetc_get_rx_buff(rx_ring, i, size);
616 skb_add_rx_frag(skb, skb_shinfo(skb)->nr_frags, rx_swbd->page,
617 rx_swbd->page_offset, size, ENETC_RXB_TRUESIZE);
619 enetc_put_rx_buff(rx_ring, rx_swbd);
622 #define ENETC_RXBD_BUNDLE 16 /* # of BDs to update at once */
624 static int enetc_clean_rx_ring(struct enetc_bdr *rx_ring,
625 struct napi_struct *napi, int work_limit)
627 int rx_frm_cnt = 0, rx_byte_cnt = 0;
630 cleaned_cnt = enetc_bd_unused(rx_ring);
631 /* next descriptor to process */
632 i = rx_ring->next_to_clean;
634 while (likely(rx_frm_cnt < work_limit)) {
635 union enetc_rx_bd *rxbd;
640 if (cleaned_cnt >= ENETC_RXBD_BUNDLE) {
641 int count = enetc_refill_rx_ring(rx_ring, cleaned_cnt);
643 cleaned_cnt -= count;
646 rxbd = ENETC_RXBD(*rx_ring, i);
647 bd_status = le32_to_cpu(rxbd->r.lstatus);
651 enetc_wr_reg(rx_ring->idr, BIT(rx_ring->index));
652 dma_rmb(); /* for reading other rxbd fields */
653 size = le16_to_cpu(rxbd->r.buf_len);
654 skb = enetc_map_rx_buff_to_skb(rx_ring, i, size);
658 enetc_get_offloads(rx_ring, rxbd, skb);
663 if (unlikely(i == rx_ring->bd_count)) {
665 rxbd = ENETC_RXBD(*rx_ring, 0);
668 if (unlikely(bd_status &
669 ENETC_RXBD_LSTATUS(ENETC_RXBD_ERR_MASK))) {
671 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
673 bd_status = le32_to_cpu(rxbd->r.lstatus);
676 if (unlikely(i == rx_ring->bd_count)) {
678 rxbd = ENETC_RXBD(*rx_ring, 0);
682 rx_ring->ndev->stats.rx_dropped++;
683 rx_ring->ndev->stats.rx_errors++;
688 /* not last BD in frame? */
689 while (!(bd_status & ENETC_RXBD_LSTATUS_F)) {
690 bd_status = le32_to_cpu(rxbd->r.lstatus);
691 size = ENETC_RXB_DMA_SIZE;
693 if (bd_status & ENETC_RXBD_LSTATUS_F) {
695 size = le16_to_cpu(rxbd->r.buf_len);
698 enetc_add_rx_buff_to_skb(rx_ring, i, size, skb);
703 if (unlikely(i == rx_ring->bd_count)) {
705 rxbd = ENETC_RXBD(*rx_ring, 0);
709 rx_byte_cnt += skb->len;
711 enetc_process_skb(rx_ring, skb);
713 napi_gro_receive(napi, skb);
718 rx_ring->next_to_clean = i;
720 rx_ring->stats.packets += rx_frm_cnt;
721 rx_ring->stats.bytes += rx_byte_cnt;
726 /* Probing and Init */
727 #define ENETC_MAX_RFS_SIZE 64
728 void enetc_get_si_caps(struct enetc_si *si)
730 struct enetc_hw *hw = &si->hw;
733 /* find out how many of various resources we have to work with */
734 val = enetc_rd(hw, ENETC_SICAPR0);
735 si->num_rx_rings = (val >> 16) & 0xff;
736 si->num_tx_rings = val & 0xff;
738 val = enetc_rd(hw, ENETC_SIRFSCAPR);
739 si->num_fs_entries = ENETC_SIRFSCAPR_GET_NUM_RFS(val);
740 si->num_fs_entries = min(si->num_fs_entries, ENETC_MAX_RFS_SIZE);
743 val = enetc_rd(hw, ENETC_SIPCAPR0);
744 if (val & ENETC_SIPCAPR0_RSS) {
745 val = enetc_rd(hw, ENETC_SIRSSCAPR);
746 si->num_rss = ENETC_SIRSSCAPR_GET_NUM_RSS(val);
750 static int enetc_dma_alloc_bdr(struct enetc_bdr *r, size_t bd_size)
752 r->bd_base = dma_alloc_coherent(r->dev, r->bd_count * bd_size,
753 &r->bd_dma_base, GFP_KERNEL);
757 /* h/w requires 128B alignment */
758 if (!IS_ALIGNED(r->bd_dma_base, 128)) {
759 dma_free_coherent(r->dev, r->bd_count * bd_size, r->bd_base,
767 static int enetc_alloc_txbdr(struct enetc_bdr *txr)
771 txr->tx_swbd = vzalloc(txr->bd_count * sizeof(struct enetc_tx_swbd));
775 err = enetc_dma_alloc_bdr(txr, sizeof(union enetc_tx_bd));
781 txr->next_to_clean = 0;
782 txr->next_to_use = 0;
787 static void enetc_free_txbdr(struct enetc_bdr *txr)
791 for (i = 0; i < txr->bd_count; i++)
792 enetc_free_tx_skb(txr, &txr->tx_swbd[i]);
794 size = txr->bd_count * sizeof(union enetc_tx_bd);
796 dma_free_coherent(txr->dev, size, txr->bd_base, txr->bd_dma_base);
803 static int enetc_alloc_tx_resources(struct enetc_ndev_priv *priv)
807 for (i = 0; i < priv->num_tx_rings; i++) {
808 err = enetc_alloc_txbdr(priv->tx_ring[i]);
818 enetc_free_txbdr(priv->tx_ring[i]);
823 static void enetc_free_tx_resources(struct enetc_ndev_priv *priv)
827 for (i = 0; i < priv->num_tx_rings; i++)
828 enetc_free_txbdr(priv->tx_ring[i]);
831 static int enetc_alloc_rxbdr(struct enetc_bdr *rxr)
835 rxr->rx_swbd = vzalloc(rxr->bd_count * sizeof(struct enetc_rx_swbd));
839 err = enetc_dma_alloc_bdr(rxr, sizeof(union enetc_rx_bd));
845 rxr->next_to_clean = 0;
846 rxr->next_to_use = 0;
847 rxr->next_to_alloc = 0;
852 static void enetc_free_rxbdr(struct enetc_bdr *rxr)
856 size = rxr->bd_count * sizeof(union enetc_rx_bd);
858 dma_free_coherent(rxr->dev, size, rxr->bd_base, rxr->bd_dma_base);
865 static int enetc_alloc_rx_resources(struct enetc_ndev_priv *priv)
869 for (i = 0; i < priv->num_rx_rings; i++) {
870 err = enetc_alloc_rxbdr(priv->rx_ring[i]);
880 enetc_free_rxbdr(priv->rx_ring[i]);
885 static void enetc_free_rx_resources(struct enetc_ndev_priv *priv)
889 for (i = 0; i < priv->num_rx_rings; i++)
890 enetc_free_rxbdr(priv->rx_ring[i]);
893 static void enetc_free_tx_ring(struct enetc_bdr *tx_ring)
897 if (!tx_ring->tx_swbd)
900 for (i = 0; i < tx_ring->bd_count; i++) {
901 struct enetc_tx_swbd *tx_swbd = &tx_ring->tx_swbd[i];
903 enetc_free_tx_skb(tx_ring, tx_swbd);
906 tx_ring->next_to_clean = 0;
907 tx_ring->next_to_use = 0;
910 static void enetc_free_rx_ring(struct enetc_bdr *rx_ring)
914 if (!rx_ring->rx_swbd)
917 for (i = 0; i < rx_ring->bd_count; i++) {
918 struct enetc_rx_swbd *rx_swbd = &rx_ring->rx_swbd[i];
923 dma_unmap_page(rx_ring->dev, rx_swbd->dma,
924 PAGE_SIZE, DMA_FROM_DEVICE);
925 __free_page(rx_swbd->page);
926 rx_swbd->page = NULL;
929 rx_ring->next_to_clean = 0;
930 rx_ring->next_to_use = 0;
931 rx_ring->next_to_alloc = 0;
934 static void enetc_free_rxtx_rings(struct enetc_ndev_priv *priv)
938 for (i = 0; i < priv->num_rx_rings; i++)
939 enetc_free_rx_ring(priv->rx_ring[i]);
941 for (i = 0; i < priv->num_tx_rings; i++)
942 enetc_free_tx_ring(priv->tx_ring[i]);
945 static int enetc_alloc_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
947 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
949 cbdr->bd_base = dma_alloc_coherent(dev, size, &cbdr->bd_dma_base,
954 /* h/w requires 128B alignment */
955 if (!IS_ALIGNED(cbdr->bd_dma_base, 128)) {
956 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
960 cbdr->next_to_clean = 0;
961 cbdr->next_to_use = 0;
966 static void enetc_free_cbdr(struct device *dev, struct enetc_cbdr *cbdr)
968 int size = cbdr->bd_count * sizeof(struct enetc_cbd);
970 dma_free_coherent(dev, size, cbdr->bd_base, cbdr->bd_dma_base);
971 cbdr->bd_base = NULL;
974 static void enetc_setup_cbdr(struct enetc_hw *hw, struct enetc_cbdr *cbdr)
976 /* set CBDR cache attributes */
977 enetc_wr(hw, ENETC_SICAR2,
978 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
980 enetc_wr(hw, ENETC_SICBDRBAR0, lower_32_bits(cbdr->bd_dma_base));
981 enetc_wr(hw, ENETC_SICBDRBAR1, upper_32_bits(cbdr->bd_dma_base));
982 enetc_wr(hw, ENETC_SICBDRLENR, ENETC_RTBLENR_LEN(cbdr->bd_count));
984 enetc_wr(hw, ENETC_SICBDRPIR, 0);
985 enetc_wr(hw, ENETC_SICBDRCIR, 0);
988 enetc_wr(hw, ENETC_SICBDRMR, BIT(31));
990 cbdr->pir = hw->reg + ENETC_SICBDRPIR;
991 cbdr->cir = hw->reg + ENETC_SICBDRCIR;
994 static void enetc_clear_cbdr(struct enetc_hw *hw)
996 enetc_wr(hw, ENETC_SICBDRMR, 0);
999 static int enetc_setup_default_rss_table(struct enetc_si *si, int num_groups)
1004 rss_table = kmalloc_array(si->num_rss, sizeof(*rss_table), GFP_KERNEL);
1008 /* Set up RSS table defaults */
1009 for (i = 0; i < si->num_rss; i++)
1010 rss_table[i] = i % num_groups;
1012 enetc_set_rss_table(si, rss_table, si->num_rss);
1019 static int enetc_configure_si(struct enetc_ndev_priv *priv)
1021 struct enetc_si *si = priv->si;
1022 struct enetc_hw *hw = &si->hw;
1025 enetc_setup_cbdr(hw, &si->cbd_ring);
1026 /* set SI cache attributes */
1027 enetc_wr(hw, ENETC_SICAR0,
1028 ENETC_SICAR_RD_COHERENT | ENETC_SICAR_WR_COHERENT);
1029 enetc_wr(hw, ENETC_SICAR1, ENETC_SICAR_MSI);
1031 enetc_wr(hw, ENETC_SIMR, ENETC_SIMR_EN);
1034 err = enetc_setup_default_rss_table(si, priv->num_rx_rings);
1042 void enetc_init_si_rings_params(struct enetc_ndev_priv *priv)
1044 struct enetc_si *si = priv->si;
1045 int cpus = num_online_cpus();
1047 priv->tx_bd_count = ENETC_BDR_DEFAULT_SIZE;
1048 priv->rx_bd_count = ENETC_BDR_DEFAULT_SIZE;
1050 /* Enable all available TX rings in order to configure as many
1051 * priorities as possible, when needed.
1052 * TODO: Make # of TX rings run-time configurable
1054 priv->num_rx_rings = min_t(int, cpus, si->num_rx_rings);
1055 priv->num_tx_rings = si->num_tx_rings;
1056 priv->bdr_int_num = cpus;
1059 si->cbd_ring.bd_count = ENETC_CBDR_DEFAULT_SIZE;
1062 int enetc_alloc_si_resources(struct enetc_ndev_priv *priv)
1064 struct enetc_si *si = priv->si;
1067 err = enetc_alloc_cbdr(priv->dev, &si->cbd_ring);
1071 priv->cls_rules = kcalloc(si->num_fs_entries, sizeof(*priv->cls_rules),
1073 if (!priv->cls_rules) {
1078 err = enetc_configure_si(priv);
1085 kfree(priv->cls_rules);
1087 enetc_clear_cbdr(&si->hw);
1088 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1093 void enetc_free_si_resources(struct enetc_ndev_priv *priv)
1095 struct enetc_si *si = priv->si;
1097 enetc_clear_cbdr(&si->hw);
1098 enetc_free_cbdr(priv->dev, &si->cbd_ring);
1100 kfree(priv->cls_rules);
1103 static void enetc_setup_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1105 int idx = tx_ring->index;
1108 enetc_txbdr_wr(hw, idx, ENETC_TBBAR0,
1109 lower_32_bits(tx_ring->bd_dma_base));
1111 enetc_txbdr_wr(hw, idx, ENETC_TBBAR1,
1112 upper_32_bits(tx_ring->bd_dma_base));
1114 WARN_ON(!IS_ALIGNED(tx_ring->bd_count, 64)); /* multiple of 64 */
1115 enetc_txbdr_wr(hw, idx, ENETC_TBLENR,
1116 ENETC_RTBLENR_LEN(tx_ring->bd_count));
1118 /* clearing PI/CI registers for Tx not supported, adjust sw indexes */
1119 tx_ring->next_to_use = enetc_txbdr_rd(hw, idx, ENETC_TBPIR);
1120 tx_ring->next_to_clean = enetc_txbdr_rd(hw, idx, ENETC_TBCIR);
1122 /* enable Tx ints by setting pkt thr to 1 */
1123 enetc_txbdr_wr(hw, idx, ENETC_TBICIR0, ENETC_TBICIR0_ICEN | 0x1);
1125 tbmr = ENETC_TBMR_EN;
1126 if (tx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_TX)
1127 tbmr |= ENETC_TBMR_VIH;
1130 enetc_txbdr_wr(hw, idx, ENETC_TBMR, tbmr);
1132 tx_ring->tpir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBPIR);
1133 tx_ring->tcir = hw->reg + ENETC_BDR(TX, idx, ENETC_TBCIR);
1134 tx_ring->idr = hw->reg + ENETC_SITXIDR;
1137 static void enetc_setup_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1139 int idx = rx_ring->index;
1142 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR0,
1143 lower_32_bits(rx_ring->bd_dma_base));
1145 enetc_rxbdr_wr(hw, idx, ENETC_RBBAR1,
1146 upper_32_bits(rx_ring->bd_dma_base));
1148 WARN_ON(!IS_ALIGNED(rx_ring->bd_count, 64)); /* multiple of 64 */
1149 enetc_rxbdr_wr(hw, idx, ENETC_RBLENR,
1150 ENETC_RTBLENR_LEN(rx_ring->bd_count));
1152 enetc_rxbdr_wr(hw, idx, ENETC_RBBSR, ENETC_RXB_DMA_SIZE);
1154 enetc_rxbdr_wr(hw, idx, ENETC_RBPIR, 0);
1156 /* enable Rx ints by setting pkt thr to 1 */
1157 enetc_rxbdr_wr(hw, idx, ENETC_RBICIR0, ENETC_RBICIR0_ICEN | 0x1);
1159 rbmr = ENETC_RBMR_EN;
1160 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1161 rbmr |= ENETC_RBMR_BDS;
1163 if (rx_ring->ndev->features & NETIF_F_HW_VLAN_CTAG_RX)
1164 rbmr |= ENETC_RBMR_VTE;
1166 rx_ring->rcir = hw->reg + ENETC_BDR(RX, idx, ENETC_RBCIR);
1167 rx_ring->idr = hw->reg + ENETC_SIRXIDR;
1169 enetc_refill_rx_ring(rx_ring, enetc_bd_unused(rx_ring));
1172 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, rbmr);
1175 static void enetc_setup_bdrs(struct enetc_ndev_priv *priv)
1179 for (i = 0; i < priv->num_tx_rings; i++)
1180 enetc_setup_txbdr(&priv->si->hw, priv->tx_ring[i]);
1182 for (i = 0; i < priv->num_rx_rings; i++)
1183 enetc_setup_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1186 static void enetc_clear_rxbdr(struct enetc_hw *hw, struct enetc_bdr *rx_ring)
1188 int idx = rx_ring->index;
1190 /* disable EN bit on ring */
1191 enetc_rxbdr_wr(hw, idx, ENETC_RBMR, 0);
1194 static void enetc_clear_txbdr(struct enetc_hw *hw, struct enetc_bdr *tx_ring)
1196 int delay = 8, timeout = 100;
1197 int idx = tx_ring->index;
1199 /* disable EN bit on ring */
1200 enetc_txbdr_wr(hw, idx, ENETC_TBMR, 0);
1202 /* wait for busy to clear */
1203 while (delay < timeout &&
1204 enetc_txbdr_rd(hw, idx, ENETC_TBSR) & ENETC_TBSR_BUSY) {
1209 if (delay >= timeout)
1210 netdev_warn(tx_ring->ndev, "timeout for tx ring #%d clear\n",
1214 static void enetc_clear_bdrs(struct enetc_ndev_priv *priv)
1218 for (i = 0; i < priv->num_tx_rings; i++)
1219 enetc_clear_txbdr(&priv->si->hw, priv->tx_ring[i]);
1221 for (i = 0; i < priv->num_rx_rings; i++)
1222 enetc_clear_rxbdr(&priv->si->hw, priv->rx_ring[i]);
1227 static int enetc_setup_irqs(struct enetc_ndev_priv *priv)
1229 struct pci_dev *pdev = priv->si->pdev;
1233 for (i = 0; i < priv->bdr_int_num; i++) {
1234 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1235 struct enetc_int_vector *v = priv->int_vector[i];
1236 int entry = ENETC_BDR_INT_BASE_IDX + i;
1237 struct enetc_hw *hw = &priv->si->hw;
1239 snprintf(v->name, sizeof(v->name), "%s-rxtx%d",
1240 priv->ndev->name, i);
1241 err = request_irq(irq, enetc_msix, 0, v->name, v);
1243 dev_err(priv->dev, "request_irq() failed!\n");
1247 v->tbier_base = hw->reg + ENETC_BDR(TX, 0, ENETC_TBIER);
1248 v->rbier = hw->reg + ENETC_BDR(RX, i, ENETC_RBIER);
1250 enetc_wr(hw, ENETC_SIMSIRRV(i), entry);
1252 for (j = 0; j < v->count_tx_rings; j++) {
1253 int idx = v->tx_ring[j].index;
1255 enetc_wr(hw, ENETC_SIMSITRV(idx), entry);
1257 cpumask_clear(&cpu_mask);
1258 cpumask_set_cpu(i % num_online_cpus(), &cpu_mask);
1259 irq_set_affinity_hint(irq, &cpu_mask);
1266 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1268 irq_set_affinity_hint(irq, NULL);
1269 free_irq(irq, priv->int_vector[i]);
1275 static void enetc_free_irqs(struct enetc_ndev_priv *priv)
1277 struct pci_dev *pdev = priv->si->pdev;
1280 for (i = 0; i < priv->bdr_int_num; i++) {
1281 int irq = pci_irq_vector(pdev, ENETC_BDR_INT_BASE_IDX + i);
1283 irq_set_affinity_hint(irq, NULL);
1284 free_irq(irq, priv->int_vector[i]);
1288 static void enetc_enable_interrupts(struct enetc_ndev_priv *priv)
1292 /* enable Tx & Rx event indication */
1293 for (i = 0; i < priv->num_rx_rings; i++) {
1294 enetc_rxbdr_wr(&priv->si->hw, i,
1295 ENETC_RBIER, ENETC_RBIER_RXTIE);
1298 for (i = 0; i < priv->num_tx_rings; i++) {
1299 enetc_txbdr_wr(&priv->si->hw, i,
1300 ENETC_TBIER, ENETC_TBIER_TXTIE);
1304 static void enetc_disable_interrupts(struct enetc_ndev_priv *priv)
1308 for (i = 0; i < priv->num_tx_rings; i++)
1309 enetc_txbdr_wr(&priv->si->hw, i, ENETC_TBIER, 0);
1311 for (i = 0; i < priv->num_rx_rings; i++)
1312 enetc_rxbdr_wr(&priv->si->hw, i, ENETC_RBIER, 0);
1315 static void adjust_link(struct net_device *ndev)
1317 struct phy_device *phydev = ndev->phydev;
1319 phy_print_status(phydev);
1322 static int enetc_phy_connect(struct net_device *ndev)
1324 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1325 struct phy_device *phydev;
1327 if (!priv->phy_node)
1328 return 0; /* phy-less mode */
1330 phydev = of_phy_connect(ndev, priv->phy_node, &adjust_link,
1333 dev_err(&ndev->dev, "could not attach to PHY\n");
1337 phy_attached_info(phydev);
1342 int enetc_open(struct net_device *ndev)
1344 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1347 err = enetc_setup_irqs(priv);
1351 err = enetc_phy_connect(ndev);
1353 goto err_phy_connect;
1355 err = enetc_alloc_tx_resources(priv);
1359 err = enetc_alloc_rx_resources(priv);
1363 enetc_setup_bdrs(priv);
1365 err = netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1367 goto err_set_queues;
1369 err = netif_set_real_num_rx_queues(ndev, priv->num_rx_rings);
1371 goto err_set_queues;
1373 for (i = 0; i < priv->bdr_int_num; i++)
1374 napi_enable(&priv->int_vector[i]->napi);
1376 enetc_enable_interrupts(priv);
1379 phy_start(ndev->phydev);
1381 netif_carrier_on(ndev);
1383 netif_tx_start_all_queues(ndev);
1388 enetc_free_rx_resources(priv);
1390 enetc_free_tx_resources(priv);
1393 phy_disconnect(ndev->phydev);
1395 enetc_free_irqs(priv);
1400 int enetc_close(struct net_device *ndev)
1402 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1405 netif_tx_stop_all_queues(ndev);
1408 phy_stop(ndev->phydev);
1409 phy_disconnect(ndev->phydev);
1411 netif_carrier_off(ndev);
1414 for (i = 0; i < priv->bdr_int_num; i++) {
1415 napi_synchronize(&priv->int_vector[i]->napi);
1416 napi_disable(&priv->int_vector[i]->napi);
1419 enetc_disable_interrupts(priv);
1420 enetc_clear_bdrs(priv);
1422 enetc_free_rxtx_rings(priv);
1423 enetc_free_rx_resources(priv);
1424 enetc_free_tx_resources(priv);
1425 enetc_free_irqs(priv);
1430 int enetc_setup_tc_mqprio(struct net_device *ndev, void *type_data)
1432 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1433 struct tc_mqprio_qopt *mqprio = type_data;
1434 struct enetc_bdr *tx_ring;
1438 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS;
1439 num_tc = mqprio->num_tc;
1442 netdev_reset_tc(ndev);
1443 netif_set_real_num_tx_queues(ndev, priv->num_tx_rings);
1445 /* Reset all ring priorities to 0 */
1446 for (i = 0; i < priv->num_tx_rings; i++) {
1447 tx_ring = priv->tx_ring[i];
1448 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, 0);
1454 /* Check if we have enough BD rings available to accommodate all TCs */
1455 if (num_tc > priv->num_tx_rings) {
1456 netdev_err(ndev, "Max %d traffic classes supported\n",
1457 priv->num_tx_rings);
1461 /* For the moment, we use only one BD ring per TC.
1463 * Configure num_tc BD rings with increasing priorities.
1465 for (i = 0; i < num_tc; i++) {
1466 tx_ring = priv->tx_ring[i];
1467 enetc_set_bdr_prio(&priv->si->hw, tx_ring->index, i);
1470 /* Reset the number of netdev queues based on the TC count */
1471 netif_set_real_num_tx_queues(ndev, num_tc);
1473 netdev_set_num_tc(ndev, num_tc);
1475 /* Each TC is associated with one netdev queue */
1476 for (i = 0; i < num_tc; i++)
1477 netdev_set_tc_queue(ndev, i, 1, i);
1482 int enetc_setup_tc(struct net_device *ndev, enum tc_setup_type type,
1486 case TC_SETUP_QDISC_MQPRIO:
1487 return enetc_setup_tc_mqprio(ndev, type_data);
1488 case TC_SETUP_QDISC_TAPRIO:
1489 return enetc_setup_tc_taprio(ndev, type_data);
1495 struct net_device_stats *enetc_get_stats(struct net_device *ndev)
1497 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1498 struct net_device_stats *stats = &ndev->stats;
1499 unsigned long packets = 0, bytes = 0;
1502 for (i = 0; i < priv->num_rx_rings; i++) {
1503 packets += priv->rx_ring[i]->stats.packets;
1504 bytes += priv->rx_ring[i]->stats.bytes;
1507 stats->rx_packets = packets;
1508 stats->rx_bytes = bytes;
1512 for (i = 0; i < priv->num_tx_rings; i++) {
1513 packets += priv->tx_ring[i]->stats.packets;
1514 bytes += priv->tx_ring[i]->stats.bytes;
1517 stats->tx_packets = packets;
1518 stats->tx_bytes = bytes;
1523 static int enetc_set_rss(struct net_device *ndev, int en)
1525 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1526 struct enetc_hw *hw = &priv->si->hw;
1529 enetc_wr(hw, ENETC_SIRBGCR, priv->num_rx_rings);
1531 reg = enetc_rd(hw, ENETC_SIMR);
1532 reg &= ~ENETC_SIMR_RSSE;
1533 reg |= (en) ? ENETC_SIMR_RSSE : 0;
1534 enetc_wr(hw, ENETC_SIMR, reg);
1539 int enetc_set_features(struct net_device *ndev,
1540 netdev_features_t features)
1542 netdev_features_t changed = ndev->features ^ features;
1544 if (changed & NETIF_F_RXHASH)
1545 enetc_set_rss(ndev, !!(features & NETIF_F_RXHASH));
1550 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1551 static int enetc_hwtstamp_set(struct net_device *ndev, struct ifreq *ifr)
1553 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1554 struct hwtstamp_config config;
1556 if (copy_from_user(&config, ifr->ifr_data, sizeof(config)))
1559 switch (config.tx_type) {
1560 case HWTSTAMP_TX_OFF:
1561 priv->active_offloads &= ~ENETC_F_TX_TSTAMP;
1563 case HWTSTAMP_TX_ON:
1564 priv->active_offloads |= ENETC_F_TX_TSTAMP;
1570 switch (config.rx_filter) {
1571 case HWTSTAMP_FILTER_NONE:
1572 priv->active_offloads &= ~ENETC_F_RX_TSTAMP;
1575 priv->active_offloads |= ENETC_F_RX_TSTAMP;
1576 config.rx_filter = HWTSTAMP_FILTER_ALL;
1579 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1583 static int enetc_hwtstamp_get(struct net_device *ndev, struct ifreq *ifr)
1585 struct enetc_ndev_priv *priv = netdev_priv(ndev);
1586 struct hwtstamp_config config;
1590 if (priv->active_offloads & ENETC_F_TX_TSTAMP)
1591 config.tx_type = HWTSTAMP_TX_ON;
1593 config.tx_type = HWTSTAMP_TX_OFF;
1595 config.rx_filter = (priv->active_offloads & ENETC_F_RX_TSTAMP) ?
1596 HWTSTAMP_FILTER_ALL : HWTSTAMP_FILTER_NONE;
1598 return copy_to_user(ifr->ifr_data, &config, sizeof(config)) ?
1603 int enetc_ioctl(struct net_device *ndev, struct ifreq *rq, int cmd)
1605 #ifdef CONFIG_FSL_ENETC_HW_TIMESTAMPING
1606 if (cmd == SIOCSHWTSTAMP)
1607 return enetc_hwtstamp_set(ndev, rq);
1608 if (cmd == SIOCGHWTSTAMP)
1609 return enetc_hwtstamp_get(ndev, rq);
1614 return phy_mii_ioctl(ndev->phydev, rq, cmd);
1617 int enetc_alloc_msix(struct enetc_ndev_priv *priv)
1619 struct pci_dev *pdev = priv->si->pdev;
1620 int size, v_tx_rings;
1621 int i, n, err, nvec;
1623 nvec = ENETC_BDR_INT_BASE_IDX + priv->bdr_int_num;
1624 /* allocate MSIX for both messaging and Rx/Tx interrupts */
1625 n = pci_alloc_irq_vectors(pdev, nvec, nvec, PCI_IRQ_MSIX);
1633 /* # of tx rings per int vector */
1634 v_tx_rings = priv->num_tx_rings / priv->bdr_int_num;
1635 size = sizeof(struct enetc_int_vector) +
1636 sizeof(struct enetc_bdr) * v_tx_rings;
1638 for (i = 0; i < priv->bdr_int_num; i++) {
1639 struct enetc_int_vector *v;
1640 struct enetc_bdr *bdr;
1643 v = kzalloc(size, GFP_KERNEL);
1649 priv->int_vector[i] = v;
1651 netif_napi_add(priv->ndev, &v->napi, enetc_poll,
1653 v->count_tx_rings = v_tx_rings;
1655 for (j = 0; j < v_tx_rings; j++) {
1658 /* default tx ring mapping policy */
1659 if (priv->bdr_int_num == ENETC_MAX_BDR_INT)
1660 idx = 2 * j + i; /* 2 CPUs */
1662 idx = j + i * v_tx_rings; /* default */
1664 __set_bit(idx, &v->tx_rings_map);
1665 bdr = &v->tx_ring[j];
1667 bdr->ndev = priv->ndev;
1668 bdr->dev = priv->dev;
1669 bdr->bd_count = priv->tx_bd_count;
1670 priv->tx_ring[idx] = bdr;
1675 bdr->ndev = priv->ndev;
1676 bdr->dev = priv->dev;
1677 bdr->bd_count = priv->rx_bd_count;
1678 priv->rx_ring[i] = bdr;
1685 netif_napi_del(&priv->int_vector[i]->napi);
1686 kfree(priv->int_vector[i]);
1689 pci_free_irq_vectors(pdev);
1694 void enetc_free_msix(struct enetc_ndev_priv *priv)
1698 for (i = 0; i < priv->bdr_int_num; i++) {
1699 struct enetc_int_vector *v = priv->int_vector[i];
1701 netif_napi_del(&v->napi);
1704 for (i = 0; i < priv->num_rx_rings; i++)
1705 priv->rx_ring[i] = NULL;
1707 for (i = 0; i < priv->num_tx_rings; i++)
1708 priv->tx_ring[i] = NULL;
1710 for (i = 0; i < priv->bdr_int_num; i++) {
1711 kfree(priv->int_vector[i]);
1712 priv->int_vector[i] = NULL;
1715 /* disable all MSIX for this device */
1716 pci_free_irq_vectors(priv->si->pdev);
1719 static void enetc_kfree_si(struct enetc_si *si)
1721 char *p = (char *)si - si->pad;
1726 static void enetc_detect_errata(struct enetc_si *si)
1728 if (si->pdev->revision == ENETC_REV1)
1729 si->errata = ENETC_ERR_TXCSUM | ENETC_ERR_VLAN_ISOL |
1733 int enetc_pci_probe(struct pci_dev *pdev, const char *name, int sizeof_priv)
1735 struct enetc_si *si, *p;
1736 struct enetc_hw *hw;
1741 err = pci_enable_device_mem(pdev);
1743 dev_err(&pdev->dev, "device enable failed\n");
1747 /* set up for high or low dma */
1748 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(64));
1750 err = dma_set_mask_and_coherent(&pdev->dev, DMA_BIT_MASK(32));
1753 "DMA configuration failed: 0x%x\n", err);
1758 err = pci_request_mem_regions(pdev, name);
1760 dev_err(&pdev->dev, "pci_request_regions failed err=%d\n", err);
1761 goto err_pci_mem_reg;
1764 pci_set_master(pdev);
1766 alloc_size = sizeof(struct enetc_si);
1768 /* align priv to 32B */
1769 alloc_size = ALIGN(alloc_size, ENETC_SI_ALIGN);
1770 alloc_size += sizeof_priv;
1772 /* force 32B alignment for enetc_si */
1773 alloc_size += ENETC_SI_ALIGN - 1;
1775 p = kzalloc(alloc_size, GFP_KERNEL);
1781 si = PTR_ALIGN(p, ENETC_SI_ALIGN);
1782 si->pad = (char *)si - (char *)p;
1784 pci_set_drvdata(pdev, si);
1788 len = pci_resource_len(pdev, ENETC_BAR_REGS);
1789 hw->reg = ioremap(pci_resource_start(pdev, ENETC_BAR_REGS), len);
1792 dev_err(&pdev->dev, "ioremap() failed\n");
1795 if (len > ENETC_PORT_BASE)
1796 hw->port = hw->reg + ENETC_PORT_BASE;
1797 if (len > ENETC_GLOBAL_BASE)
1798 hw->global = hw->reg + ENETC_GLOBAL_BASE;
1800 enetc_detect_errata(si);
1807 pci_release_mem_regions(pdev);
1810 pci_disable_device(pdev);
1815 void enetc_pci_remove(struct pci_dev *pdev)
1817 struct enetc_si *si = pci_get_drvdata(pdev);
1818 struct enetc_hw *hw = &si->hw;
1822 pci_release_mem_regions(pdev);
1823 pci_disable_device(pdev);