2 * This file is provided under a dual BSD/GPLv2 license. When using or
3 * redistributing this file, you may do so under either license.
7 * Copyright(c) 2012 Intel Corporation. All rights reserved.
8 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
10 * This program is free software; you can redistribute it and/or modify
11 * it under the terms of version 2 of the GNU General Public License as
12 * published by the Free Software Foundation.
16 * Copyright(c) 2012 Intel Corporation. All rights reserved.
17 * Copyright (C) 2015 EMC Corporation. All Rights Reserved.
19 * Redistribution and use in source and binary forms, with or without
20 * modification, are permitted provided that the following conditions
23 * * Redistributions of source code must retain the above copyright
24 * notice, this list of conditions and the following disclaimer.
25 * * Redistributions in binary form must reproduce the above copy
26 * notice, this list of conditions and the following disclaimer in
27 * the documentation and/or other materials provided with the
29 * * Neither the name of Intel Corporation nor the names of its
30 * contributors may be used to endorse or promote products derived
31 * from this software without specific prior written permission.
33 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
34 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
35 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
36 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
37 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
38 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
39 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
40 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
41 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
42 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
43 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
45 * Intel PCIe NTB Linux driver
47 * Contact Information:
48 * Jon Mason <jon.mason@intel.com>
51 #include <linux/debugfs.h>
52 #include <linux/delay.h>
53 #include <linux/init.h>
54 #include <linux/interrupt.h>
55 #include <linux/module.h>
56 #include <linux/pci.h>
57 #include <linux/random.h>
58 #include <linux/slab.h>
59 #include <linux/ntb.h>
61 #include "ntb_hw_intel.h"
63 #define NTB_NAME "ntb_hw_intel"
64 #define NTB_DESC "Intel(R) PCI-E Non-Transparent Bridge Driver"
67 MODULE_DESCRIPTION(NTB_DESC);
68 MODULE_VERSION(NTB_VER);
69 MODULE_LICENSE("Dual BSD/GPL");
70 MODULE_AUTHOR("Intel Corporation");
72 #define bar0_off(base, bar) ((base) + ((bar) << 2))
73 #define bar2_off(base, bar) bar0_off(base, (bar) - 2)
75 static const struct intel_ntb_reg atom_reg;
76 static const struct intel_ntb_alt_reg atom_pri_reg;
77 static const struct intel_ntb_alt_reg atom_sec_reg;
78 static const struct intel_ntb_alt_reg atom_b2b_reg;
79 static const struct intel_ntb_xlat_reg atom_pri_xlat;
80 static const struct intel_ntb_xlat_reg atom_sec_xlat;
81 static const struct intel_ntb_reg xeon_reg;
82 static const struct intel_ntb_alt_reg xeon_pri_reg;
83 static const struct intel_ntb_alt_reg xeon_sec_reg;
84 static const struct intel_ntb_alt_reg xeon_b2b_reg;
85 static const struct intel_ntb_xlat_reg xeon_pri_xlat;
86 static const struct intel_ntb_xlat_reg xeon_sec_xlat;
87 static struct intel_b2b_addr xeon_b2b_usd_addr;
88 static struct intel_b2b_addr xeon_b2b_dsd_addr;
89 static const struct intel_ntb_reg skx_reg;
90 static const struct intel_ntb_alt_reg skx_pri_reg;
91 static const struct intel_ntb_alt_reg skx_b2b_reg;
92 static const struct intel_ntb_xlat_reg skx_sec_xlat;
93 static const struct ntb_dev_ops intel_ntb_ops;
94 static const struct ntb_dev_ops intel_ntb3_ops;
96 static const struct file_operations intel_ntb_debugfs_info;
97 static struct dentry *debugfs_dir;
99 static int b2b_mw_idx = -1;
100 module_param(b2b_mw_idx, int, 0644);
101 MODULE_PARM_DESC(b2b_mw_idx, "Use this mw idx to access the peer ntb. A "
102 "value of zero or positive starts from first mw idx, and a "
103 "negative value starts from last mw idx. Both sides MUST "
104 "set the same value here!");
106 static unsigned int b2b_mw_share;
107 module_param(b2b_mw_share, uint, 0644);
108 MODULE_PARM_DESC(b2b_mw_share, "If the b2b mw is large enough, configure the "
109 "ntb so that the peer ntb only occupies the first half of "
110 "the mw, so the second half can still be used as a mw. Both "
111 "sides MUST set the same value here!");
113 module_param_named(xeon_b2b_usd_bar2_addr64,
114 xeon_b2b_usd_addr.bar2_addr64, ullong, 0644);
115 MODULE_PARM_DESC(xeon_b2b_usd_bar2_addr64,
116 "XEON B2B USD BAR 2 64-bit address");
118 module_param_named(xeon_b2b_usd_bar4_addr64,
119 xeon_b2b_usd_addr.bar4_addr64, ullong, 0644);
120 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr64,
121 "XEON B2B USD BAR 4 64-bit address");
123 module_param_named(xeon_b2b_usd_bar4_addr32,
124 xeon_b2b_usd_addr.bar4_addr32, ullong, 0644);
125 MODULE_PARM_DESC(xeon_b2b_usd_bar4_addr32,
126 "XEON B2B USD split-BAR 4 32-bit address");
128 module_param_named(xeon_b2b_usd_bar5_addr32,
129 xeon_b2b_usd_addr.bar5_addr32, ullong, 0644);
130 MODULE_PARM_DESC(xeon_b2b_usd_bar5_addr32,
131 "XEON B2B USD split-BAR 5 32-bit address");
133 module_param_named(xeon_b2b_dsd_bar2_addr64,
134 xeon_b2b_dsd_addr.bar2_addr64, ullong, 0644);
135 MODULE_PARM_DESC(xeon_b2b_dsd_bar2_addr64,
136 "XEON B2B DSD BAR 2 64-bit address");
138 module_param_named(xeon_b2b_dsd_bar4_addr64,
139 xeon_b2b_dsd_addr.bar4_addr64, ullong, 0644);
140 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr64,
141 "XEON B2B DSD BAR 4 64-bit address");
143 module_param_named(xeon_b2b_dsd_bar4_addr32,
144 xeon_b2b_dsd_addr.bar4_addr32, ullong, 0644);
145 MODULE_PARM_DESC(xeon_b2b_dsd_bar4_addr32,
146 "XEON B2B DSD split-BAR 4 32-bit address");
148 module_param_named(xeon_b2b_dsd_bar5_addr32,
149 xeon_b2b_dsd_addr.bar5_addr32, ullong, 0644);
150 MODULE_PARM_DESC(xeon_b2b_dsd_bar5_addr32,
151 "XEON B2B DSD split-BAR 5 32-bit address");
153 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd);
154 static int xeon_init_isr(struct intel_ntb_dev *ndev);
158 #define ioread64 readq
160 #define ioread64 _ioread64
161 static inline u64 _ioread64(void __iomem *mmio)
165 low = ioread32(mmio);
166 high = ioread32(mmio + sizeof(u32));
167 return low | (high << 32);
174 #define iowrite64 writeq
176 #define iowrite64 _iowrite64
177 static inline void _iowrite64(u64 val, void __iomem *mmio)
179 iowrite32(val, mmio);
180 iowrite32(val >> 32, mmio + sizeof(u32));
185 static inline int pdev_is_atom(struct pci_dev *pdev)
187 switch (pdev->device) {
188 case PCI_DEVICE_ID_INTEL_NTB_B2B_BWD:
194 static inline int pdev_is_xeon(struct pci_dev *pdev)
196 switch (pdev->device) {
197 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
198 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
199 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
200 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
201 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
202 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
203 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
204 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
205 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
206 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
207 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
208 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
209 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
210 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
211 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
217 static inline int pdev_is_skx_xeon(struct pci_dev *pdev)
219 if (pdev->device == PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)
225 static inline void ndev_reset_unsafe_flags(struct intel_ntb_dev *ndev)
227 ndev->unsafe_flags = 0;
228 ndev->unsafe_flags_ignore = 0;
230 /* Only B2B has a workaround to avoid SDOORBELL */
231 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP)
232 if (!ntb_topo_is_b2b(ndev->ntb.topo))
233 ndev->unsafe_flags |= NTB_UNSAFE_DB;
235 /* No low level workaround to avoid SB01BASE */
236 if (ndev->hwerr_flags & NTB_HWERR_SB01BASE_LOCKUP) {
237 ndev->unsafe_flags |= NTB_UNSAFE_DB;
238 ndev->unsafe_flags |= NTB_UNSAFE_SPAD;
242 static inline int ndev_is_unsafe(struct intel_ntb_dev *ndev,
245 return !!(flag & ndev->unsafe_flags & ~ndev->unsafe_flags_ignore);
248 static inline int ndev_ignore_unsafe(struct intel_ntb_dev *ndev,
251 flag &= ndev->unsafe_flags;
252 ndev->unsafe_flags_ignore |= flag;
257 static int ndev_mw_to_bar(struct intel_ntb_dev *ndev, int idx)
259 if (idx < 0 || idx >= ndev->mw_count)
261 return ndev->reg->mw_bar[idx];
264 static inline int ndev_db_addr(struct intel_ntb_dev *ndev,
265 phys_addr_t *db_addr, resource_size_t *db_size,
266 phys_addr_t reg_addr, unsigned long reg)
268 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
269 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
272 *db_addr = reg_addr + reg;
273 dev_dbg(ndev_dev(ndev), "Peer db addr %llx\n", *db_addr);
277 *db_size = ndev->reg->db_size;
278 dev_dbg(ndev_dev(ndev), "Peer db size %llx\n", *db_size);
284 static inline u64 ndev_db_read(struct intel_ntb_dev *ndev,
287 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
288 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
290 return ndev->reg->db_ioread(mmio);
293 static inline int ndev_db_write(struct intel_ntb_dev *ndev, u64 db_bits,
296 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
297 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
299 if (db_bits & ~ndev->db_valid_mask)
302 ndev->reg->db_iowrite(db_bits, mmio);
307 static inline int ndev_db_set_mask(struct intel_ntb_dev *ndev, u64 db_bits,
310 unsigned long irqflags;
312 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
313 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
315 if (db_bits & ~ndev->db_valid_mask)
318 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
320 ndev->db_mask |= db_bits;
321 ndev->reg->db_iowrite(ndev->db_mask, mmio);
323 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
328 static inline int ndev_db_clear_mask(struct intel_ntb_dev *ndev, u64 db_bits,
331 unsigned long irqflags;
333 if (ndev_is_unsafe(ndev, NTB_UNSAFE_DB))
334 pr_warn_once("%s: NTB unsafe doorbell access", __func__);
336 if (db_bits & ~ndev->db_valid_mask)
339 spin_lock_irqsave(&ndev->db_mask_lock, irqflags);
341 ndev->db_mask &= ~db_bits;
342 ndev->reg->db_iowrite(ndev->db_mask, mmio);
344 spin_unlock_irqrestore(&ndev->db_mask_lock, irqflags);
349 static inline int ndev_vec_mask(struct intel_ntb_dev *ndev, int db_vector)
353 shift = ndev->db_vec_shift;
354 mask = BIT_ULL(shift) - 1;
356 return mask << (shift * db_vector);
359 static inline int ndev_spad_addr(struct intel_ntb_dev *ndev, int idx,
360 phys_addr_t *spad_addr, phys_addr_t reg_addr,
363 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
364 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
366 if (idx < 0 || idx >= ndev->spad_count)
370 *spad_addr = reg_addr + reg + (idx << 2);
371 dev_dbg(ndev_dev(ndev), "Peer spad addr %llx\n", *spad_addr);
377 static inline u32 ndev_spad_read(struct intel_ntb_dev *ndev, int idx,
380 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
381 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
383 if (idx < 0 || idx >= ndev->spad_count)
386 return ioread32(mmio + (idx << 2));
389 static inline int ndev_spad_write(struct intel_ntb_dev *ndev, int idx, u32 val,
392 if (ndev_is_unsafe(ndev, NTB_UNSAFE_SPAD))
393 pr_warn_once("%s: NTB unsafe scratchpad access", __func__);
395 if (idx < 0 || idx >= ndev->spad_count)
398 iowrite32(val, mmio + (idx << 2));
403 static irqreturn_t ndev_interrupt(struct intel_ntb_dev *ndev, int vec)
407 vec_mask = ndev_vec_mask(ndev, vec);
409 if ((ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) && (vec == 31))
410 vec_mask |= ndev->db_link_mask;
412 dev_dbg(ndev_dev(ndev), "vec %d vec_mask %llx\n", vec, vec_mask);
414 ndev->last_ts = jiffies;
416 if (vec_mask & ndev->db_link_mask) {
417 if (ndev->reg->poll_link(ndev))
418 ntb_link_event(&ndev->ntb);
421 if (vec_mask & ndev->db_valid_mask)
422 ntb_db_event(&ndev->ntb, vec);
427 static irqreturn_t ndev_vec_isr(int irq, void *dev)
429 struct intel_ntb_vec *nvec = dev;
431 dev_dbg(ndev_dev(nvec->ndev), "irq: %d nvec->num: %d\n",
434 return ndev_interrupt(nvec->ndev, nvec->num);
437 static irqreturn_t ndev_irq_isr(int irq, void *dev)
439 struct intel_ntb_dev *ndev = dev;
441 return ndev_interrupt(ndev, irq - ndev_pdev(ndev)->irq);
444 static int ndev_init_isr(struct intel_ntb_dev *ndev,
445 int msix_min, int msix_max,
446 int msix_shift, int total_shift)
448 struct pci_dev *pdev;
449 int rc, i, msix_count, node;
451 pdev = ndev_pdev(ndev);
453 node = dev_to_node(&pdev->dev);
455 /* Mask all doorbell interrupts */
456 ndev->db_mask = ndev->db_valid_mask;
457 ndev->reg->db_iowrite(ndev->db_mask,
459 ndev->self_reg->db_mask);
461 /* Try to set up msix irq */
463 ndev->vec = kzalloc_node(msix_max * sizeof(*ndev->vec),
466 goto err_msix_vec_alloc;
468 ndev->msix = kzalloc_node(msix_max * sizeof(*ndev->msix),
473 for (i = 0; i < msix_max; ++i)
474 ndev->msix[i].entry = i;
476 msix_count = pci_enable_msix_range(pdev, ndev->msix,
479 goto err_msix_enable;
481 for (i = 0; i < msix_count; ++i) {
482 ndev->vec[i].ndev = ndev;
483 ndev->vec[i].num = i;
484 rc = request_irq(ndev->msix[i].vector, ndev_vec_isr, 0,
485 "ndev_vec_isr", &ndev->vec[i]);
487 goto err_msix_request;
490 dev_dbg(ndev_dev(ndev), "Using %d msix interrupts\n", msix_count);
491 ndev->db_vec_count = msix_count;
492 ndev->db_vec_shift = msix_shift;
497 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
498 pci_disable_msix(pdev);
507 /* Try to set up msi irq */
509 rc = pci_enable_msi(pdev);
513 rc = request_irq(pdev->irq, ndev_irq_isr, 0,
514 "ndev_irq_isr", ndev);
516 goto err_msi_request;
518 dev_dbg(ndev_dev(ndev), "Using msi interrupts\n");
519 ndev->db_vec_count = 1;
520 ndev->db_vec_shift = total_shift;
524 pci_disable_msi(pdev);
527 /* Try to set up intx irq */
531 rc = request_irq(pdev->irq, ndev_irq_isr, IRQF_SHARED,
532 "ndev_irq_isr", ndev);
534 goto err_intx_request;
536 dev_dbg(ndev_dev(ndev), "Using intx interrupts\n");
537 ndev->db_vec_count = 1;
538 ndev->db_vec_shift = total_shift;
545 static void ndev_deinit_isr(struct intel_ntb_dev *ndev)
547 struct pci_dev *pdev;
550 pdev = ndev_pdev(ndev);
552 /* Mask all doorbell interrupts */
553 ndev->db_mask = ndev->db_valid_mask;
554 ndev->reg->db_iowrite(ndev->db_mask,
556 ndev->self_reg->db_mask);
559 i = ndev->db_vec_count;
561 free_irq(ndev->msix[i].vector, &ndev->vec[i]);
562 pci_disable_msix(pdev);
566 free_irq(pdev->irq, ndev);
567 if (pci_dev_msi_enabled(pdev))
568 pci_disable_msi(pdev);
572 static ssize_t ndev_ntb3_debugfs_read(struct file *filp, char __user *ubuf,
573 size_t count, loff_t *offp)
575 struct intel_ntb_dev *ndev;
580 union { u64 v64; u32 v32; u16 v16; } u;
582 ndev = filp->private_data;
583 mmio = ndev->self_mmio;
585 buf_size = min(count, 0x800ul);
587 buf = kmalloc(buf_size, GFP_KERNEL);
593 off += scnprintf(buf + off, buf_size - off,
594 "NTB Device Information:\n");
596 off += scnprintf(buf + off, buf_size - off,
597 "Connection Topology -\t%s\n",
598 ntb_topo_string(ndev->ntb.topo));
600 off += scnprintf(buf + off, buf_size - off,
601 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
602 off += scnprintf(buf + off, buf_size - off,
603 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
605 if (!ndev->reg->link_is_up(ndev))
606 off += scnprintf(buf + off, buf_size - off,
607 "Link Status -\t\tDown\n");
609 off += scnprintf(buf + off, buf_size - off,
610 "Link Status -\t\tUp\n");
611 off += scnprintf(buf + off, buf_size - off,
612 "Link Speed -\t\tPCI-E Gen %u\n",
613 NTB_LNK_STA_SPEED(ndev->lnk_sta));
614 off += scnprintf(buf + off, buf_size - off,
615 "Link Width -\t\tx%u\n",
616 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
619 off += scnprintf(buf + off, buf_size - off,
620 "Memory Window Count -\t%u\n", ndev->mw_count);
621 off += scnprintf(buf + off, buf_size - off,
622 "Scratchpad Count -\t%u\n", ndev->spad_count);
623 off += scnprintf(buf + off, buf_size - off,
624 "Doorbell Count -\t%u\n", ndev->db_count);
625 off += scnprintf(buf + off, buf_size - off,
626 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
627 off += scnprintf(buf + off, buf_size - off,
628 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
630 off += scnprintf(buf + off, buf_size - off,
631 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
632 off += scnprintf(buf + off, buf_size - off,
633 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
634 off += scnprintf(buf + off, buf_size - off,
635 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
637 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
638 off += scnprintf(buf + off, buf_size - off,
639 "Doorbell Mask -\t\t%#llx\n", u.v64);
641 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
642 off += scnprintf(buf + off, buf_size - off,
643 "Doorbell Bell -\t\t%#llx\n", u.v64);
645 off += scnprintf(buf + off, buf_size - off,
646 "\nNTB Incoming XLAT:\n");
648 u.v64 = ioread64(mmio + SKX_IMBAR1XBASE_OFFSET);
649 off += scnprintf(buf + off, buf_size - off,
650 "IMBAR1XBASE -\t\t%#018llx\n", u.v64);
652 u.v64 = ioread64(mmio + SKX_IMBAR2XBASE_OFFSET);
653 off += scnprintf(buf + off, buf_size - off,
654 "IMBAR2XBASE -\t\t%#018llx\n", u.v64);
656 u.v64 = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
657 off += scnprintf(buf + off, buf_size - off,
658 "IMBAR1XLMT -\t\t\t%#018llx\n", u.v64);
660 u.v64 = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
661 off += scnprintf(buf + off, buf_size - off,
662 "IMBAR2XLMT -\t\t\t%#018llx\n", u.v64);
664 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
665 off += scnprintf(buf + off, buf_size - off,
666 "\nNTB Outgoing B2B XLAT:\n");
668 u.v64 = ioread64(mmio + SKX_EMBAR1XBASE_OFFSET);
669 off += scnprintf(buf + off, buf_size - off,
670 "EMBAR1XBASE -\t\t%#018llx\n", u.v64);
672 u.v64 = ioread64(mmio + SKX_EMBAR2XBASE_OFFSET);
673 off += scnprintf(buf + off, buf_size - off,
674 "EMBAR2XBASE -\t\t%#018llx\n", u.v64);
676 u.v64 = ioread64(mmio + SKX_EMBAR1XLMT_OFFSET);
677 off += scnprintf(buf + off, buf_size - off,
678 "EMBAR1XLMT -\t\t%#018llx\n", u.v64);
680 u.v64 = ioread64(mmio + SKX_EMBAR2XLMT_OFFSET);
681 off += scnprintf(buf + off, buf_size - off,
682 "EMBAR2XLMT -\t\t%#018llx\n", u.v64);
684 off += scnprintf(buf + off, buf_size - off,
685 "\nNTB Secondary BAR:\n");
687 u.v64 = ioread64(mmio + SKX_EMBAR0_OFFSET);
688 off += scnprintf(buf + off, buf_size - off,
689 "EMBAR0 -\t\t%#018llx\n", u.v64);
691 u.v64 = ioread64(mmio + SKX_EMBAR1_OFFSET);
692 off += scnprintf(buf + off, buf_size - off,
693 "EMBAR1 -\t\t%#018llx\n", u.v64);
695 u.v64 = ioread64(mmio + SKX_EMBAR2_OFFSET);
696 off += scnprintf(buf + off, buf_size - off,
697 "EMBAR2 -\t\t%#018llx\n", u.v64);
700 off += scnprintf(buf + off, buf_size - off,
701 "\nNTB Statistics:\n");
703 u.v16 = ioread16(mmio + SKX_USMEMMISS_OFFSET);
704 off += scnprintf(buf + off, buf_size - off,
705 "Upstream Memory Miss -\t%u\n", u.v16);
707 off += scnprintf(buf + off, buf_size - off,
708 "\nNTB Hardware Errors:\n");
710 if (!pci_read_config_word(ndev->ntb.pdev,
711 SKX_DEVSTS_OFFSET, &u.v16))
712 off += scnprintf(buf + off, buf_size - off,
713 "DEVSTS -\t\t%#06x\n", u.v16);
715 if (!pci_read_config_word(ndev->ntb.pdev,
716 SKX_LINK_STATUS_OFFSET, &u.v16))
717 off += scnprintf(buf + off, buf_size - off,
718 "LNKSTS -\t\t%#06x\n", u.v16);
720 if (!pci_read_config_dword(ndev->ntb.pdev,
721 SKX_UNCERRSTS_OFFSET, &u.v32))
722 off += scnprintf(buf + off, buf_size - off,
723 "UNCERRSTS -\t\t%#06x\n", u.v32);
725 if (!pci_read_config_dword(ndev->ntb.pdev,
726 SKX_CORERRSTS_OFFSET, &u.v32))
727 off += scnprintf(buf + off, buf_size - off,
728 "CORERRSTS -\t\t%#06x\n", u.v32);
730 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
735 static ssize_t ndev_ntb_debugfs_read(struct file *filp, char __user *ubuf,
736 size_t count, loff_t *offp)
738 struct intel_ntb_dev *ndev;
739 struct pci_dev *pdev;
744 union { u64 v64; u32 v32; u16 v16; u8 v8; } u;
746 ndev = filp->private_data;
747 pdev = ndev_pdev(ndev);
748 mmio = ndev->self_mmio;
750 buf_size = min(count, 0x800ul);
752 buf = kmalloc(buf_size, GFP_KERNEL);
758 off += scnprintf(buf + off, buf_size - off,
759 "NTB Device Information:\n");
761 off += scnprintf(buf + off, buf_size - off,
762 "Connection Topology -\t%s\n",
763 ntb_topo_string(ndev->ntb.topo));
765 if (ndev->b2b_idx != UINT_MAX) {
766 off += scnprintf(buf + off, buf_size - off,
767 "B2B MW Idx -\t\t%u\n", ndev->b2b_idx);
768 off += scnprintf(buf + off, buf_size - off,
769 "B2B Offset -\t\t%#lx\n", ndev->b2b_off);
772 off += scnprintf(buf + off, buf_size - off,
773 "BAR4 Split -\t\t%s\n",
774 ndev->bar4_split ? "yes" : "no");
776 off += scnprintf(buf + off, buf_size - off,
777 "NTB CTL -\t\t%#06x\n", ndev->ntb_ctl);
778 off += scnprintf(buf + off, buf_size - off,
779 "LNK STA -\t\t%#06x\n", ndev->lnk_sta);
781 if (!ndev->reg->link_is_up(ndev)) {
782 off += scnprintf(buf + off, buf_size - off,
783 "Link Status -\t\tDown\n");
785 off += scnprintf(buf + off, buf_size - off,
786 "Link Status -\t\tUp\n");
787 off += scnprintf(buf + off, buf_size - off,
788 "Link Speed -\t\tPCI-E Gen %u\n",
789 NTB_LNK_STA_SPEED(ndev->lnk_sta));
790 off += scnprintf(buf + off, buf_size - off,
791 "Link Width -\t\tx%u\n",
792 NTB_LNK_STA_WIDTH(ndev->lnk_sta));
795 off += scnprintf(buf + off, buf_size - off,
796 "Memory Window Count -\t%u\n", ndev->mw_count);
797 off += scnprintf(buf + off, buf_size - off,
798 "Scratchpad Count -\t%u\n", ndev->spad_count);
799 off += scnprintf(buf + off, buf_size - off,
800 "Doorbell Count -\t%u\n", ndev->db_count);
801 off += scnprintf(buf + off, buf_size - off,
802 "Doorbell Vector Count -\t%u\n", ndev->db_vec_count);
803 off += scnprintf(buf + off, buf_size - off,
804 "Doorbell Vector Shift -\t%u\n", ndev->db_vec_shift);
806 off += scnprintf(buf + off, buf_size - off,
807 "Doorbell Valid Mask -\t%#llx\n", ndev->db_valid_mask);
808 off += scnprintf(buf + off, buf_size - off,
809 "Doorbell Link Mask -\t%#llx\n", ndev->db_link_mask);
810 off += scnprintf(buf + off, buf_size - off,
811 "Doorbell Mask Cached -\t%#llx\n", ndev->db_mask);
813 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_mask);
814 off += scnprintf(buf + off, buf_size - off,
815 "Doorbell Mask -\t\t%#llx\n", u.v64);
817 u.v64 = ndev_db_read(ndev, mmio + ndev->self_reg->db_bell);
818 off += scnprintf(buf + off, buf_size - off,
819 "Doorbell Bell -\t\t%#llx\n", u.v64);
821 off += scnprintf(buf + off, buf_size - off,
822 "\nNTB Window Size:\n");
824 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &u.v8);
825 off += scnprintf(buf + off, buf_size - off,
826 "PBAR23SZ %hhu\n", u.v8);
827 if (!ndev->bar4_split) {
828 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &u.v8);
829 off += scnprintf(buf + off, buf_size - off,
830 "PBAR45SZ %hhu\n", u.v8);
832 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &u.v8);
833 off += scnprintf(buf + off, buf_size - off,
834 "PBAR4SZ %hhu\n", u.v8);
835 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &u.v8);
836 off += scnprintf(buf + off, buf_size - off,
837 "PBAR5SZ %hhu\n", u.v8);
840 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &u.v8);
841 off += scnprintf(buf + off, buf_size - off,
842 "SBAR23SZ %hhu\n", u.v8);
843 if (!ndev->bar4_split) {
844 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &u.v8);
845 off += scnprintf(buf + off, buf_size - off,
846 "SBAR45SZ %hhu\n", u.v8);
848 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &u.v8);
849 off += scnprintf(buf + off, buf_size - off,
850 "SBAR4SZ %hhu\n", u.v8);
851 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &u.v8);
852 off += scnprintf(buf + off, buf_size - off,
853 "SBAR5SZ %hhu\n", u.v8);
856 off += scnprintf(buf + off, buf_size - off,
857 "\nNTB Incoming XLAT:\n");
859 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 2));
860 off += scnprintf(buf + off, buf_size - off,
861 "XLAT23 -\t\t%#018llx\n", u.v64);
863 if (ndev->bar4_split) {
864 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
865 off += scnprintf(buf + off, buf_size - off,
866 "XLAT4 -\t\t\t%#06x\n", u.v32);
868 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 5));
869 off += scnprintf(buf + off, buf_size - off,
870 "XLAT5 -\t\t\t%#06x\n", u.v32);
872 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_xlat, 4));
873 off += scnprintf(buf + off, buf_size - off,
874 "XLAT45 -\t\t%#018llx\n", u.v64);
877 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 2));
878 off += scnprintf(buf + off, buf_size - off,
879 "LMT23 -\t\t\t%#018llx\n", u.v64);
881 if (ndev->bar4_split) {
882 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
883 off += scnprintf(buf + off, buf_size - off,
884 "LMT4 -\t\t\t%#06x\n", u.v32);
885 u.v32 = ioread32(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 5));
886 off += scnprintf(buf + off, buf_size - off,
887 "LMT5 -\t\t\t%#06x\n", u.v32);
889 u.v64 = ioread64(mmio + bar2_off(ndev->xlat_reg->bar2_limit, 4));
890 off += scnprintf(buf + off, buf_size - off,
891 "LMT45 -\t\t\t%#018llx\n", u.v64);
894 if (pdev_is_xeon(pdev)) {
895 if (ntb_topo_is_b2b(ndev->ntb.topo)) {
896 off += scnprintf(buf + off, buf_size - off,
897 "\nNTB Outgoing B2B XLAT:\n");
899 u.v64 = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
900 off += scnprintf(buf + off, buf_size - off,
901 "B2B XLAT23 -\t\t%#018llx\n", u.v64);
903 if (ndev->bar4_split) {
904 u.v32 = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
905 off += scnprintf(buf + off, buf_size - off,
906 "B2B XLAT4 -\t\t%#06x\n",
908 u.v32 = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
909 off += scnprintf(buf + off, buf_size - off,
910 "B2B XLAT5 -\t\t%#06x\n",
913 u.v64 = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
914 off += scnprintf(buf + off, buf_size - off,
915 "B2B XLAT45 -\t\t%#018llx\n",
919 u.v64 = ioread64(mmio + XEON_PBAR23LMT_OFFSET);
920 off += scnprintf(buf + off, buf_size - off,
921 "B2B LMT23 -\t\t%#018llx\n", u.v64);
923 if (ndev->bar4_split) {
924 u.v32 = ioread32(mmio + XEON_PBAR4LMT_OFFSET);
925 off += scnprintf(buf + off, buf_size - off,
926 "B2B LMT4 -\t\t%#06x\n",
928 u.v32 = ioread32(mmio + XEON_PBAR5LMT_OFFSET);
929 off += scnprintf(buf + off, buf_size - off,
930 "B2B LMT5 -\t\t%#06x\n",
933 u.v64 = ioread64(mmio + XEON_PBAR45LMT_OFFSET);
934 off += scnprintf(buf + off, buf_size - off,
935 "B2B LMT45 -\t\t%#018llx\n",
939 off += scnprintf(buf + off, buf_size - off,
940 "\nNTB Secondary BAR:\n");
942 u.v64 = ioread64(mmio + XEON_SBAR0BASE_OFFSET);
943 off += scnprintf(buf + off, buf_size - off,
944 "SBAR01 -\t\t%#018llx\n", u.v64);
946 u.v64 = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
947 off += scnprintf(buf + off, buf_size - off,
948 "SBAR23 -\t\t%#018llx\n", u.v64);
950 if (ndev->bar4_split) {
951 u.v32 = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
952 off += scnprintf(buf + off, buf_size - off,
953 "SBAR4 -\t\t\t%#06x\n", u.v32);
954 u.v32 = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
955 off += scnprintf(buf + off, buf_size - off,
956 "SBAR5 -\t\t\t%#06x\n", u.v32);
958 u.v64 = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
959 off += scnprintf(buf + off, buf_size - off,
960 "SBAR45 -\t\t%#018llx\n",
965 off += scnprintf(buf + off, buf_size - off,
966 "\nXEON NTB Statistics:\n");
968 u.v16 = ioread16(mmio + XEON_USMEMMISS_OFFSET);
969 off += scnprintf(buf + off, buf_size - off,
970 "Upstream Memory Miss -\t%u\n", u.v16);
972 off += scnprintf(buf + off, buf_size - off,
973 "\nXEON NTB Hardware Errors:\n");
975 if (!pci_read_config_word(pdev,
976 XEON_DEVSTS_OFFSET, &u.v16))
977 off += scnprintf(buf + off, buf_size - off,
978 "DEVSTS -\t\t%#06x\n", u.v16);
980 if (!pci_read_config_word(pdev,
981 XEON_LINK_STATUS_OFFSET, &u.v16))
982 off += scnprintf(buf + off, buf_size - off,
983 "LNKSTS -\t\t%#06x\n", u.v16);
985 if (!pci_read_config_dword(pdev,
986 XEON_UNCERRSTS_OFFSET, &u.v32))
987 off += scnprintf(buf + off, buf_size - off,
988 "UNCERRSTS -\t\t%#06x\n", u.v32);
990 if (!pci_read_config_dword(pdev,
991 XEON_CORERRSTS_OFFSET, &u.v32))
992 off += scnprintf(buf + off, buf_size - off,
993 "CORERRSTS -\t\t%#06x\n", u.v32);
996 ret = simple_read_from_buffer(ubuf, count, offp, buf, off);
1001 static ssize_t ndev_debugfs_read(struct file *filp, char __user *ubuf,
1002 size_t count, loff_t *offp)
1004 struct intel_ntb_dev *ndev = filp->private_data;
1006 if (pdev_is_xeon(ndev->ntb.pdev) ||
1007 pdev_is_atom(ndev->ntb.pdev))
1008 return ndev_ntb_debugfs_read(filp, ubuf, count, offp);
1009 else if (pdev_is_skx_xeon(ndev->ntb.pdev))
1010 return ndev_ntb3_debugfs_read(filp, ubuf, count, offp);
1015 static void ndev_init_debugfs(struct intel_ntb_dev *ndev)
1018 ndev->debugfs_dir = NULL;
1019 ndev->debugfs_info = NULL;
1022 debugfs_create_dir(ndev_name(ndev), debugfs_dir);
1023 if (!ndev->debugfs_dir)
1024 ndev->debugfs_info = NULL;
1026 ndev->debugfs_info =
1027 debugfs_create_file("info", S_IRUSR,
1028 ndev->debugfs_dir, ndev,
1029 &intel_ntb_debugfs_info);
1033 static void ndev_deinit_debugfs(struct intel_ntb_dev *ndev)
1035 debugfs_remove_recursive(ndev->debugfs_dir);
1038 static int intel_ntb_mw_count(struct ntb_dev *ntb)
1040 return ntb_ndev(ntb)->mw_count;
1043 static int intel_ntb_mw_get_range(struct ntb_dev *ntb, int idx,
1045 resource_size_t *size,
1046 resource_size_t *align,
1047 resource_size_t *align_size)
1049 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1052 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1055 bar = ndev_mw_to_bar(ndev, idx);
1060 *base = pci_resource_start(ndev->ntb.pdev, bar) +
1061 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1064 *size = pci_resource_len(ndev->ntb.pdev, bar) -
1065 (idx == ndev->b2b_idx ? ndev->b2b_off : 0);
1068 *align = pci_resource_len(ndev->ntb.pdev, bar);
1076 static int intel_ntb_mw_set_trans(struct ntb_dev *ntb, int idx,
1077 dma_addr_t addr, resource_size_t size)
1079 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1080 unsigned long base_reg, xlat_reg, limit_reg;
1081 resource_size_t bar_size, mw_size;
1083 u64 base, limit, reg_val;
1086 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1089 bar = ndev_mw_to_bar(ndev, idx);
1093 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1095 if (idx == ndev->b2b_idx)
1096 mw_size = bar_size - ndev->b2b_off;
1100 /* hardware requires that addr is aligned to bar size */
1101 if (addr & (bar_size - 1))
1104 /* make sure the range fits in the usable mw size */
1108 mmio = ndev->self_mmio;
1109 base_reg = bar0_off(ndev->xlat_reg->bar0_base, bar);
1110 xlat_reg = bar2_off(ndev->xlat_reg->bar2_xlat, bar);
1111 limit_reg = bar2_off(ndev->xlat_reg->bar2_limit, bar);
1113 if (bar < 4 || !ndev->bar4_split) {
1114 base = ioread64(mmio + base_reg) & NTB_BAR_MASK_64;
1116 /* Set the limit if supported, if size is not mw_size */
1117 if (limit_reg && size != mw_size)
1118 limit = base + size;
1122 /* set and verify setting the translation address */
1123 iowrite64(addr, mmio + xlat_reg);
1124 reg_val = ioread64(mmio + xlat_reg);
1125 if (reg_val != addr) {
1126 iowrite64(0, mmio + xlat_reg);
1130 /* set and verify setting the limit */
1131 iowrite64(limit, mmio + limit_reg);
1132 reg_val = ioread64(mmio + limit_reg);
1133 if (reg_val != limit) {
1134 iowrite64(base, mmio + limit_reg);
1135 iowrite64(0, mmio + xlat_reg);
1139 /* split bar addr range must all be 32 bit */
1140 if (addr & (~0ull << 32))
1142 if ((addr + size) & (~0ull << 32))
1145 base = ioread32(mmio + base_reg) & NTB_BAR_MASK_32;
1147 /* Set the limit if supported, if size is not mw_size */
1148 if (limit_reg && size != mw_size)
1149 limit = base + size;
1153 /* set and verify setting the translation address */
1154 iowrite32(addr, mmio + xlat_reg);
1155 reg_val = ioread32(mmio + xlat_reg);
1156 if (reg_val != addr) {
1157 iowrite32(0, mmio + xlat_reg);
1161 /* set and verify setting the limit */
1162 iowrite32(limit, mmio + limit_reg);
1163 reg_val = ioread32(mmio + limit_reg);
1164 if (reg_val != limit) {
1165 iowrite32(base, mmio + limit_reg);
1166 iowrite32(0, mmio + xlat_reg);
1174 static int intel_ntb_link_is_up(struct ntb_dev *ntb,
1175 enum ntb_speed *speed,
1176 enum ntb_width *width)
1178 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1180 if (ndev->reg->link_is_up(ndev)) {
1182 *speed = NTB_LNK_STA_SPEED(ndev->lnk_sta);
1184 *width = NTB_LNK_STA_WIDTH(ndev->lnk_sta);
1187 /* TODO MAYBE: is it possible to observe the link speed and
1188 * width while link is training? */
1190 *speed = NTB_SPEED_NONE;
1192 *width = NTB_WIDTH_NONE;
1197 static int intel_ntb_link_enable(struct ntb_dev *ntb,
1198 enum ntb_speed max_speed,
1199 enum ntb_width max_width)
1201 struct intel_ntb_dev *ndev;
1204 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1206 if (ndev->ntb.topo == NTB_TOPO_SEC)
1209 dev_dbg(ndev_dev(ndev),
1210 "Enabling link with max_speed %d max_width %d\n",
1211 max_speed, max_width);
1212 if (max_speed != NTB_SPEED_AUTO)
1213 dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
1214 if (max_width != NTB_WIDTH_AUTO)
1215 dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
1217 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1218 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
1219 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
1220 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
1221 if (ndev->bar4_split)
1222 ntb_ctl |= NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP;
1223 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1228 static int intel_ntb_link_disable(struct ntb_dev *ntb)
1230 struct intel_ntb_dev *ndev;
1233 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1235 if (ndev->ntb.topo == NTB_TOPO_SEC)
1238 dev_dbg(ndev_dev(ndev), "Disabling link\n");
1240 /* Bring NTB link down */
1241 ntb_cntl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1242 ntb_cntl &= ~(NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP);
1243 ntb_cntl &= ~(NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP);
1244 if (ndev->bar4_split)
1245 ntb_cntl &= ~(NTB_CTL_P2S_BAR5_SNOOP | NTB_CTL_S2P_BAR5_SNOOP);
1246 ntb_cntl |= NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK;
1247 iowrite32(ntb_cntl, ndev->self_mmio + ndev->reg->ntb_ctl);
1252 static int intel_ntb_db_is_unsafe(struct ntb_dev *ntb)
1254 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_DB);
1257 static u64 intel_ntb_db_valid_mask(struct ntb_dev *ntb)
1259 return ntb_ndev(ntb)->db_valid_mask;
1262 static int intel_ntb_db_vector_count(struct ntb_dev *ntb)
1264 struct intel_ntb_dev *ndev;
1266 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1268 return ndev->db_vec_count;
1271 static u64 intel_ntb_db_vector_mask(struct ntb_dev *ntb, int db_vector)
1273 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1275 if (db_vector < 0 || db_vector > ndev->db_vec_count)
1278 return ndev->db_valid_mask & ndev_vec_mask(ndev, db_vector);
1281 static u64 intel_ntb_db_read(struct ntb_dev *ntb)
1283 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1285 return ndev_db_read(ndev,
1287 ndev->self_reg->db_bell);
1290 static int intel_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1292 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1294 return ndev_db_write(ndev, db_bits,
1296 ndev->self_reg->db_bell);
1299 static int intel_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1301 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1303 return ndev_db_set_mask(ndev, db_bits,
1305 ndev->self_reg->db_mask);
1308 static int intel_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1310 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1312 return ndev_db_clear_mask(ndev, db_bits,
1314 ndev->self_reg->db_mask);
1317 static int intel_ntb_peer_db_addr(struct ntb_dev *ntb,
1318 phys_addr_t *db_addr,
1319 resource_size_t *db_size)
1321 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1323 return ndev_db_addr(ndev, db_addr, db_size, ndev->peer_addr,
1324 ndev->peer_reg->db_bell);
1327 static int intel_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1329 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1331 return ndev_db_write(ndev, db_bits,
1333 ndev->peer_reg->db_bell);
1336 static int intel_ntb_spad_is_unsafe(struct ntb_dev *ntb)
1338 return ndev_ignore_unsafe(ntb_ndev(ntb), NTB_UNSAFE_SPAD);
1341 static int intel_ntb_spad_count(struct ntb_dev *ntb)
1343 struct intel_ntb_dev *ndev;
1345 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1347 return ndev->spad_count;
1350 static u32 intel_ntb_spad_read(struct ntb_dev *ntb, int idx)
1352 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1354 return ndev_spad_read(ndev, idx,
1356 ndev->self_reg->spad);
1359 static int intel_ntb_spad_write(struct ntb_dev *ntb,
1362 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1364 return ndev_spad_write(ndev, idx, val,
1366 ndev->self_reg->spad);
1369 static int intel_ntb_peer_spad_addr(struct ntb_dev *ntb, int idx,
1370 phys_addr_t *spad_addr)
1372 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1374 return ndev_spad_addr(ndev, idx, spad_addr, ndev->peer_addr,
1375 ndev->peer_reg->spad);
1378 static u32 intel_ntb_peer_spad_read(struct ntb_dev *ntb, int idx)
1380 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1382 return ndev_spad_read(ndev, idx,
1384 ndev->peer_reg->spad);
1387 static int intel_ntb_peer_spad_write(struct ntb_dev *ntb,
1390 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1392 return ndev_spad_write(ndev, idx, val,
1394 ndev->peer_reg->spad);
1399 static u64 atom_db_ioread(void __iomem *mmio)
1401 return ioread64(mmio);
1404 static void atom_db_iowrite(u64 bits, void __iomem *mmio)
1406 iowrite64(bits, mmio);
1409 static int atom_poll_link(struct intel_ntb_dev *ndev)
1413 ntb_ctl = ioread32(ndev->self_mmio + ATOM_NTBCNTL_OFFSET);
1415 if (ntb_ctl == ndev->ntb_ctl)
1418 ndev->ntb_ctl = ntb_ctl;
1420 ndev->lnk_sta = ioread32(ndev->self_mmio + ATOM_LINK_STATUS_OFFSET);
1425 static int atom_link_is_up(struct intel_ntb_dev *ndev)
1427 return ATOM_NTB_CTL_ACTIVE(ndev->ntb_ctl);
1430 static int atom_link_is_err(struct intel_ntb_dev *ndev)
1432 if (ioread32(ndev->self_mmio + ATOM_LTSSMSTATEJMP_OFFSET)
1433 & ATOM_LTSSMSTATEJMP_FORCEDETECT)
1436 if (ioread32(ndev->self_mmio + ATOM_IBSTERRRCRVSTS0_OFFSET)
1437 & ATOM_IBIST_ERR_OFLOW)
1443 static inline enum ntb_topo atom_ppd_topo(struct intel_ntb_dev *ndev, u32 ppd)
1445 switch (ppd & ATOM_PPD_TOPO_MASK) {
1446 case ATOM_PPD_TOPO_B2B_USD:
1447 dev_dbg(ndev_dev(ndev), "PPD %d B2B USD\n", ppd);
1448 return NTB_TOPO_B2B_USD;
1450 case ATOM_PPD_TOPO_B2B_DSD:
1451 dev_dbg(ndev_dev(ndev), "PPD %d B2B DSD\n", ppd);
1452 return NTB_TOPO_B2B_DSD;
1454 case ATOM_PPD_TOPO_PRI_USD:
1455 case ATOM_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
1456 case ATOM_PPD_TOPO_SEC_USD:
1457 case ATOM_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
1458 dev_dbg(ndev_dev(ndev), "PPD %d non B2B disabled\n", ppd);
1459 return NTB_TOPO_NONE;
1462 dev_dbg(ndev_dev(ndev), "PPD %d invalid\n", ppd);
1463 return NTB_TOPO_NONE;
1466 static void atom_link_hb(struct work_struct *work)
1468 struct intel_ntb_dev *ndev = hb_ndev(work);
1469 unsigned long poll_ts;
1473 poll_ts = ndev->last_ts + ATOM_LINK_HB_TIMEOUT;
1475 /* Delay polling the link status if an interrupt was received,
1476 * unless the cached link status says the link is down.
1478 if (time_after(poll_ts, jiffies) && atom_link_is_up(ndev)) {
1479 schedule_delayed_work(&ndev->hb_timer, poll_ts - jiffies);
1483 if (atom_poll_link(ndev))
1484 ntb_link_event(&ndev->ntb);
1486 if (atom_link_is_up(ndev) || !atom_link_is_err(ndev)) {
1487 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1491 /* Link is down with error: recover the link! */
1493 mmio = ndev->self_mmio;
1495 /* Driver resets the NTB ModPhy lanes - magic! */
1496 iowrite8(0xe0, mmio + ATOM_MODPHY_PCSREG6);
1497 iowrite8(0x40, mmio + ATOM_MODPHY_PCSREG4);
1498 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG4);
1499 iowrite8(0x60, mmio + ATOM_MODPHY_PCSREG6);
1501 /* Driver waits 100ms to allow the NTB ModPhy to settle */
1504 /* Clear AER Errors, write to clear */
1505 status32 = ioread32(mmio + ATOM_ERRCORSTS_OFFSET);
1506 dev_dbg(ndev_dev(ndev), "ERRCORSTS = %x\n", status32);
1507 status32 &= PCI_ERR_COR_REP_ROLL;
1508 iowrite32(status32, mmio + ATOM_ERRCORSTS_OFFSET);
1510 /* Clear unexpected electrical idle event in LTSSM, write to clear */
1511 status32 = ioread32(mmio + ATOM_LTSSMERRSTS0_OFFSET);
1512 dev_dbg(ndev_dev(ndev), "LTSSMERRSTS0 = %x\n", status32);
1513 status32 |= ATOM_LTSSMERRSTS0_UNEXPECTEDEI;
1514 iowrite32(status32, mmio + ATOM_LTSSMERRSTS0_OFFSET);
1516 /* Clear DeSkew Buffer error, write to clear */
1517 status32 = ioread32(mmio + ATOM_DESKEWSTS_OFFSET);
1518 dev_dbg(ndev_dev(ndev), "DESKEWSTS = %x\n", status32);
1519 status32 |= ATOM_DESKEWSTS_DBERR;
1520 iowrite32(status32, mmio + ATOM_DESKEWSTS_OFFSET);
1522 status32 = ioread32(mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1523 dev_dbg(ndev_dev(ndev), "IBSTERRRCRVSTS0 = %x\n", status32);
1524 status32 &= ATOM_IBIST_ERR_OFLOW;
1525 iowrite32(status32, mmio + ATOM_IBSTERRRCRVSTS0_OFFSET);
1527 /* Releases the NTB state machine to allow the link to retrain */
1528 status32 = ioread32(mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1529 dev_dbg(ndev_dev(ndev), "LTSSMSTATEJMP = %x\n", status32);
1530 status32 &= ~ATOM_LTSSMSTATEJMP_FORCEDETECT;
1531 iowrite32(status32, mmio + ATOM_LTSSMSTATEJMP_OFFSET);
1533 /* There is a potential race between the 2 NTB devices recovering at the
1534 * same time. If the times are the same, the link will not recover and
1535 * the driver will be stuck in this loop forever. Add a random interval
1536 * to the recovery time to prevent this race.
1538 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_RECOVERY_TIME
1539 + prandom_u32() % ATOM_LINK_RECOVERY_TIME);
1542 static int atom_init_isr(struct intel_ntb_dev *ndev)
1546 rc = ndev_init_isr(ndev, 1, ATOM_DB_MSIX_VECTOR_COUNT,
1547 ATOM_DB_MSIX_VECTOR_SHIFT, ATOM_DB_TOTAL_SHIFT);
1551 /* ATOM doesn't have link status interrupt, poll on that platform */
1552 ndev->last_ts = jiffies;
1553 INIT_DELAYED_WORK(&ndev->hb_timer, atom_link_hb);
1554 schedule_delayed_work(&ndev->hb_timer, ATOM_LINK_HB_TIMEOUT);
1559 static void atom_deinit_isr(struct intel_ntb_dev *ndev)
1561 cancel_delayed_work_sync(&ndev->hb_timer);
1562 ndev_deinit_isr(ndev);
1565 static int atom_init_ntb(struct intel_ntb_dev *ndev)
1567 ndev->mw_count = ATOM_MW_COUNT;
1568 ndev->spad_count = ATOM_SPAD_COUNT;
1569 ndev->db_count = ATOM_DB_COUNT;
1571 switch (ndev->ntb.topo) {
1572 case NTB_TOPO_B2B_USD:
1573 case NTB_TOPO_B2B_DSD:
1574 ndev->self_reg = &atom_pri_reg;
1575 ndev->peer_reg = &atom_b2b_reg;
1576 ndev->xlat_reg = &atom_sec_xlat;
1578 /* Enable Bus Master and Memory Space on the secondary side */
1579 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1580 ndev->self_mmio + ATOM_SPCICMD_OFFSET);
1588 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1593 static int atom_init_dev(struct intel_ntb_dev *ndev)
1598 rc = pci_read_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET, &ppd);
1602 ndev->ntb.topo = atom_ppd_topo(ndev, ppd);
1603 if (ndev->ntb.topo == NTB_TOPO_NONE)
1606 rc = atom_init_ntb(ndev);
1610 rc = atom_init_isr(ndev);
1614 if (ndev->ntb.topo != NTB_TOPO_SEC) {
1615 /* Initiate PCI-E link training */
1616 rc = pci_write_config_dword(ndev->ntb.pdev, ATOM_PPD_OFFSET,
1617 ppd | ATOM_PPD_INIT_LINK);
1625 static void atom_deinit_dev(struct intel_ntb_dev *ndev)
1627 atom_deinit_isr(ndev);
1630 /* Skylake Xeon NTB */
1632 static int skx_poll_link(struct intel_ntb_dev *ndev)
1637 ndev->reg->db_iowrite(ndev->db_link_mask,
1639 ndev->self_reg->db_clear);
1641 rc = pci_read_config_word(ndev->ntb.pdev,
1642 SKX_LINK_STATUS_OFFSET, ®_val);
1646 if (reg_val == ndev->lnk_sta)
1649 ndev->lnk_sta = reg_val;
1654 static u64 skx_db_ioread(void __iomem *mmio)
1656 return ioread64(mmio);
1659 static void skx_db_iowrite(u64 bits, void __iomem *mmio)
1661 iowrite64(bits, mmio);
1664 static int skx_init_isr(struct intel_ntb_dev *ndev)
1669 * The MSIX vectors and the interrupt status bits are not lined up
1670 * on Skylake. By default the link status bit is bit 32, however it
1671 * is by default MSIX vector0. We need to fixup to line them up.
1672 * The vectors at reset is 1-32,0. We need to reprogram to 0-32.
1675 for (i = 0; i < SKX_DB_MSIX_VECTOR_COUNT; i++)
1676 iowrite8(i, ndev->self_mmio + SKX_INTVEC_OFFSET + i);
1678 /* move link status down one as workaround */
1679 if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD) {
1680 iowrite8(SKX_DB_MSIX_VECTOR_COUNT - 2,
1681 ndev->self_mmio + SKX_INTVEC_OFFSET +
1682 (SKX_DB_MSIX_VECTOR_COUNT - 1));
1685 return ndev_init_isr(ndev, SKX_DB_MSIX_VECTOR_COUNT,
1686 SKX_DB_MSIX_VECTOR_COUNT,
1687 SKX_DB_MSIX_VECTOR_SHIFT,
1688 SKX_DB_TOTAL_SHIFT);
1691 static int skx_setup_b2b_mw(struct intel_ntb_dev *ndev,
1692 const struct intel_b2b_addr *addr,
1693 const struct intel_b2b_addr *peer_addr)
1695 struct pci_dev *pdev;
1697 resource_size_t bar_size;
1698 phys_addr_t bar_addr;
1702 pdev = ndev_pdev(ndev);
1703 mmio = ndev->self_mmio;
1705 if (ndev->b2b_idx == UINT_MAX) {
1706 dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
1710 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
1714 dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
1716 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
1718 dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
1720 if (b2b_mw_share && ((bar_size >> 1) >= XEON_B2B_MIN_SIZE)) {
1721 dev_dbg(ndev_dev(ndev),
1722 "b2b using first half of bar\n");
1723 ndev->b2b_off = bar_size >> 1;
1724 } else if (bar_size >= XEON_B2B_MIN_SIZE) {
1725 dev_dbg(ndev_dev(ndev),
1726 "b2b using whole bar\n");
1730 dev_dbg(ndev_dev(ndev),
1731 "b2b bar size is too small\n");
1737 * Reset the secondary bar sizes to match the primary bar sizes,
1738 * except disable or halve the size of the b2b secondary bar.
1740 pci_read_config_byte(pdev, SKX_IMBAR1SZ_OFFSET, &bar_sz);
1741 dev_dbg(ndev_dev(ndev), "IMBAR1SZ %#x\n", bar_sz);
1749 pci_write_config_byte(pdev, SKX_EMBAR1SZ_OFFSET, bar_sz);
1750 pci_read_config_byte(pdev, SKX_EMBAR1SZ_OFFSET, &bar_sz);
1751 dev_dbg(ndev_dev(ndev), "EMBAR1SZ %#x\n", bar_sz);
1753 pci_read_config_byte(pdev, SKX_IMBAR2SZ_OFFSET, &bar_sz);
1754 dev_dbg(ndev_dev(ndev), "IMBAR2SZ %#x\n", bar_sz);
1762 pci_write_config_byte(pdev, SKX_EMBAR2SZ_OFFSET, bar_sz);
1763 pci_read_config_byte(pdev, SKX_EMBAR2SZ_OFFSET, &bar_sz);
1764 dev_dbg(ndev_dev(ndev), "EMBAR2SZ %#x\n", bar_sz);
1766 /* SBAR01 hit by first part of the b2b bar */
1768 bar_addr = addr->bar0_addr;
1769 else if (b2b_bar == 1)
1770 bar_addr = addr->bar2_addr64;
1771 else if (b2b_bar == 2)
1772 bar_addr = addr->bar4_addr64;
1776 /* setup incoming bar limits == base addrs (zero length windows) */
1777 bar_addr = addr->bar2_addr64 + (b2b_bar == 1 ? ndev->b2b_off : 0);
1778 iowrite64(bar_addr, mmio + SKX_IMBAR1XLMT_OFFSET);
1779 bar_addr = ioread64(mmio + SKX_IMBAR1XLMT_OFFSET);
1780 dev_dbg(ndev_dev(ndev), "IMBAR1XLMT %#018llx\n", bar_addr);
1782 bar_addr = addr->bar4_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
1783 iowrite64(bar_addr, mmio + SKX_IMBAR2XLMT_OFFSET);
1784 bar_addr = ioread64(mmio + SKX_IMBAR2XLMT_OFFSET);
1785 dev_dbg(ndev_dev(ndev), "IMBAR2XLMT %#018llx\n", bar_addr);
1787 /* zero incoming translation addrs */
1788 iowrite64(0, mmio + SKX_IMBAR1XBASE_OFFSET);
1789 iowrite64(0, mmio + SKX_IMBAR2XBASE_OFFSET);
1791 ndev->peer_mmio = ndev->self_mmio;
1796 static int skx_init_ntb(struct intel_ntb_dev *ndev)
1801 ndev->mw_count = XEON_MW_COUNT;
1802 ndev->spad_count = SKX_SPAD_COUNT;
1803 ndev->db_count = SKX_DB_COUNT;
1804 ndev->db_link_mask = SKX_DB_LINK_BIT;
1806 /* DB fixup for using 31 right now */
1807 if (ndev->hwerr_flags & NTB_HWERR_MSIX_VECTOR32_BAD)
1808 ndev->db_link_mask |= BIT_ULL(31);
1810 switch (ndev->ntb.topo) {
1811 case NTB_TOPO_B2B_USD:
1812 case NTB_TOPO_B2B_DSD:
1813 ndev->self_reg = &skx_pri_reg;
1814 ndev->peer_reg = &skx_b2b_reg;
1815 ndev->xlat_reg = &skx_sec_xlat;
1817 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
1818 rc = skx_setup_b2b_mw(ndev,
1820 &xeon_b2b_usd_addr);
1822 rc = skx_setup_b2b_mw(ndev,
1824 &xeon_b2b_dsd_addr);
1830 /* Enable Bus Master and Memory Space on the secondary side */
1831 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
1832 ndev->self_mmio + SKX_SPCICMD_OFFSET);
1840 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
1842 ndev->reg->db_iowrite(ndev->db_valid_mask,
1844 ndev->self_reg->db_mask);
1849 static int skx_init_dev(struct intel_ntb_dev *ndev)
1851 struct pci_dev *pdev;
1855 pdev = ndev_pdev(ndev);
1857 ndev->reg = &skx_reg;
1859 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
1863 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
1864 dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
1865 ntb_topo_string(ndev->ntb.topo));
1866 if (ndev->ntb.topo == NTB_TOPO_NONE)
1869 if (pdev_is_skx_xeon(pdev))
1870 ndev->hwerr_flags |= NTB_HWERR_MSIX_VECTOR32_BAD;
1872 rc = skx_init_ntb(ndev);
1876 return skx_init_isr(ndev);
1879 static int intel_ntb3_link_enable(struct ntb_dev *ntb,
1880 enum ntb_speed max_speed,
1881 enum ntb_width max_width)
1883 struct intel_ntb_dev *ndev;
1886 ndev = container_of(ntb, struct intel_ntb_dev, ntb);
1888 dev_dbg(ndev_dev(ndev),
1889 "Enabling link with max_speed %d max_width %d\n",
1890 max_speed, max_width);
1892 if (max_speed != NTB_SPEED_AUTO)
1893 dev_dbg(ndev_dev(ndev), "ignoring max_speed %d\n", max_speed);
1894 if (max_width != NTB_WIDTH_AUTO)
1895 dev_dbg(ndev_dev(ndev), "ignoring max_width %d\n", max_width);
1897 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
1898 ntb_ctl &= ~(NTB_CTL_DISABLE | NTB_CTL_CFG_LOCK);
1899 ntb_ctl |= NTB_CTL_P2S_BAR2_SNOOP | NTB_CTL_S2P_BAR2_SNOOP;
1900 ntb_ctl |= NTB_CTL_P2S_BAR4_SNOOP | NTB_CTL_S2P_BAR4_SNOOP;
1901 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
1905 static int intel_ntb3_mw_set_trans(struct ntb_dev *ntb, int idx,
1906 dma_addr_t addr, resource_size_t size)
1908 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1909 unsigned long xlat_reg, limit_reg;
1910 resource_size_t bar_size, mw_size;
1912 u64 base, limit, reg_val;
1915 if (idx >= ndev->b2b_idx && !ndev->b2b_off)
1918 bar = ndev_mw_to_bar(ndev, idx);
1922 bar_size = pci_resource_len(ndev->ntb.pdev, bar);
1924 if (idx == ndev->b2b_idx)
1925 mw_size = bar_size - ndev->b2b_off;
1929 /* hardware requires that addr is aligned to bar size */
1930 if (addr & (bar_size - 1))
1933 /* make sure the range fits in the usable mw size */
1937 mmio = ndev->self_mmio;
1938 xlat_reg = ndev->xlat_reg->bar2_xlat + (idx * 0x10);
1939 limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10);
1940 base = pci_resource_start(ndev->ntb.pdev, bar);
1942 /* Set the limit if supported, if size is not mw_size */
1943 if (limit_reg && size != mw_size)
1944 limit = base + size;
1946 limit = base + mw_size;
1948 /* set and verify setting the translation address */
1949 iowrite64(addr, mmio + xlat_reg);
1950 reg_val = ioread64(mmio + xlat_reg);
1951 if (reg_val != addr) {
1952 iowrite64(0, mmio + xlat_reg);
1956 dev_dbg(ndev_dev(ndev), "BAR %d IMBARXBASE: %#Lx\n", bar, reg_val);
1958 /* set and verify setting the limit */
1959 iowrite64(limit, mmio + limit_reg);
1960 reg_val = ioread64(mmio + limit_reg);
1961 if (reg_val != limit) {
1962 iowrite64(base, mmio + limit_reg);
1963 iowrite64(0, mmio + xlat_reg);
1967 dev_dbg(ndev_dev(ndev), "BAR %d IMBARXLMT: %#Lx\n", bar, reg_val);
1970 limit_reg = ndev->xlat_reg->bar2_limit + (idx * 0x10) + 0x4000;
1971 base = ioread64(mmio + SKX_EMBAR1_OFFSET + (8 * idx));
1974 if (limit_reg && size != mw_size)
1975 limit = base + size;
1977 limit = base + mw_size;
1979 /* set and verify setting the limit */
1980 iowrite64(limit, mmio + limit_reg);
1981 reg_val = ioread64(mmio + limit_reg);
1982 if (reg_val != limit) {
1983 iowrite64(base, mmio + limit_reg);
1984 iowrite64(0, mmio + xlat_reg);
1988 dev_dbg(ndev_dev(ndev), "BAR %d EMBARXLMT: %#Lx\n", bar, reg_val);
1993 static int intel_ntb3_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1995 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
1998 if (db_bits & ~ndev->db_valid_mask)
2002 bit = __ffs(db_bits);
2003 iowrite32(1, ndev->peer_mmio +
2004 ndev->peer_reg->db_bell + (bit * 4));
2005 db_bits &= db_bits - 1;
2011 static u64 intel_ntb3_db_read(struct ntb_dev *ntb)
2013 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2015 return ndev_db_read(ndev,
2017 ndev->self_reg->db_clear);
2020 static int intel_ntb3_db_clear(struct ntb_dev *ntb, u64 db_bits)
2022 struct intel_ntb_dev *ndev = ntb_ndev(ntb);
2024 return ndev_db_write(ndev, db_bits,
2026 ndev->self_reg->db_clear);
2031 static u64 xeon_db_ioread(void __iomem *mmio)
2033 return (u64)ioread16(mmio);
2036 static void xeon_db_iowrite(u64 bits, void __iomem *mmio)
2038 iowrite16((u16)bits, mmio);
2041 static int xeon_poll_link(struct intel_ntb_dev *ndev)
2046 ndev->reg->db_iowrite(ndev->db_link_mask,
2048 ndev->self_reg->db_bell);
2050 rc = pci_read_config_word(ndev->ntb.pdev,
2051 XEON_LINK_STATUS_OFFSET, ®_val);
2055 if (reg_val == ndev->lnk_sta)
2058 ndev->lnk_sta = reg_val;
2063 static int xeon_link_is_up(struct intel_ntb_dev *ndev)
2065 if (ndev->ntb.topo == NTB_TOPO_SEC)
2068 return NTB_LNK_STA_ACTIVE(ndev->lnk_sta);
2071 static inline enum ntb_topo xeon_ppd_topo(struct intel_ntb_dev *ndev, u8 ppd)
2073 switch (ppd & XEON_PPD_TOPO_MASK) {
2074 case XEON_PPD_TOPO_B2B_USD:
2075 return NTB_TOPO_B2B_USD;
2077 case XEON_PPD_TOPO_B2B_DSD:
2078 return NTB_TOPO_B2B_DSD;
2080 case XEON_PPD_TOPO_PRI_USD:
2081 case XEON_PPD_TOPO_PRI_DSD: /* accept bogus PRI_DSD */
2082 return NTB_TOPO_PRI;
2084 case XEON_PPD_TOPO_SEC_USD:
2085 case XEON_PPD_TOPO_SEC_DSD: /* accept bogus SEC_DSD */
2086 return NTB_TOPO_SEC;
2089 return NTB_TOPO_NONE;
2092 static inline int xeon_ppd_bar4_split(struct intel_ntb_dev *ndev, u8 ppd)
2094 if (ppd & XEON_PPD_SPLIT_BAR_MASK) {
2095 dev_dbg(ndev_dev(ndev), "PPD %d split bar\n", ppd);
2101 static int xeon_init_isr(struct intel_ntb_dev *ndev)
2103 return ndev_init_isr(ndev, XEON_DB_MSIX_VECTOR_COUNT,
2104 XEON_DB_MSIX_VECTOR_COUNT,
2105 XEON_DB_MSIX_VECTOR_SHIFT,
2106 XEON_DB_TOTAL_SHIFT);
2109 static void xeon_deinit_isr(struct intel_ntb_dev *ndev)
2111 ndev_deinit_isr(ndev);
2114 static int xeon_setup_b2b_mw(struct intel_ntb_dev *ndev,
2115 const struct intel_b2b_addr *addr,
2116 const struct intel_b2b_addr *peer_addr)
2118 struct pci_dev *pdev;
2120 resource_size_t bar_size;
2121 phys_addr_t bar_addr;
2125 pdev = ndev_pdev(ndev);
2126 mmio = ndev->self_mmio;
2128 if (ndev->b2b_idx == UINT_MAX) {
2129 dev_dbg(ndev_dev(ndev), "not using b2b mw\n");
2133 b2b_bar = ndev_mw_to_bar(ndev, ndev->b2b_idx);
2137 dev_dbg(ndev_dev(ndev), "using b2b mw bar %d\n", b2b_bar);
2139 bar_size = pci_resource_len(ndev->ntb.pdev, b2b_bar);
2141 dev_dbg(ndev_dev(ndev), "b2b bar size %#llx\n", bar_size);
2143 if (b2b_mw_share && XEON_B2B_MIN_SIZE <= bar_size >> 1) {
2144 dev_dbg(ndev_dev(ndev),
2145 "b2b using first half of bar\n");
2146 ndev->b2b_off = bar_size >> 1;
2147 } else if (XEON_B2B_MIN_SIZE <= bar_size) {
2148 dev_dbg(ndev_dev(ndev),
2149 "b2b using whole bar\n");
2153 dev_dbg(ndev_dev(ndev),
2154 "b2b bar size is too small\n");
2159 /* Reset the secondary bar sizes to match the primary bar sizes,
2160 * except disable or halve the size of the b2b secondary bar.
2162 * Note: code for each specific bar size register, because the register
2163 * offsets are not in a consistent order (bar5sz comes after ppd, odd).
2165 pci_read_config_byte(pdev, XEON_PBAR23SZ_OFFSET, &bar_sz);
2166 dev_dbg(ndev_dev(ndev), "PBAR23SZ %#x\n", bar_sz);
2173 pci_write_config_byte(pdev, XEON_SBAR23SZ_OFFSET, bar_sz);
2174 pci_read_config_byte(pdev, XEON_SBAR23SZ_OFFSET, &bar_sz);
2175 dev_dbg(ndev_dev(ndev), "SBAR23SZ %#x\n", bar_sz);
2177 if (!ndev->bar4_split) {
2178 pci_read_config_byte(pdev, XEON_PBAR45SZ_OFFSET, &bar_sz);
2179 dev_dbg(ndev_dev(ndev), "PBAR45SZ %#x\n", bar_sz);
2186 pci_write_config_byte(pdev, XEON_SBAR45SZ_OFFSET, bar_sz);
2187 pci_read_config_byte(pdev, XEON_SBAR45SZ_OFFSET, &bar_sz);
2188 dev_dbg(ndev_dev(ndev), "SBAR45SZ %#x\n", bar_sz);
2190 pci_read_config_byte(pdev, XEON_PBAR4SZ_OFFSET, &bar_sz);
2191 dev_dbg(ndev_dev(ndev), "PBAR4SZ %#x\n", bar_sz);
2198 pci_write_config_byte(pdev, XEON_SBAR4SZ_OFFSET, bar_sz);
2199 pci_read_config_byte(pdev, XEON_SBAR4SZ_OFFSET, &bar_sz);
2200 dev_dbg(ndev_dev(ndev), "SBAR4SZ %#x\n", bar_sz);
2202 pci_read_config_byte(pdev, XEON_PBAR5SZ_OFFSET, &bar_sz);
2203 dev_dbg(ndev_dev(ndev), "PBAR5SZ %#x\n", bar_sz);
2210 pci_write_config_byte(pdev, XEON_SBAR5SZ_OFFSET, bar_sz);
2211 pci_read_config_byte(pdev, XEON_SBAR5SZ_OFFSET, &bar_sz);
2212 dev_dbg(ndev_dev(ndev), "SBAR5SZ %#x\n", bar_sz);
2215 /* SBAR01 hit by first part of the b2b bar */
2217 bar_addr = addr->bar0_addr;
2218 else if (b2b_bar == 2)
2219 bar_addr = addr->bar2_addr64;
2220 else if (b2b_bar == 4 && !ndev->bar4_split)
2221 bar_addr = addr->bar4_addr64;
2222 else if (b2b_bar == 4)
2223 bar_addr = addr->bar4_addr32;
2224 else if (b2b_bar == 5)
2225 bar_addr = addr->bar5_addr32;
2229 dev_dbg(ndev_dev(ndev), "SBAR01 %#018llx\n", bar_addr);
2230 iowrite64(bar_addr, mmio + XEON_SBAR0BASE_OFFSET);
2232 /* Other SBAR are normally hit by the PBAR xlat, except for b2b bar.
2233 * The b2b bar is either disabled above, or configured half-size, and
2234 * it starts at the PBAR xlat + offset.
2237 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
2238 iowrite64(bar_addr, mmio + XEON_SBAR23BASE_OFFSET);
2239 bar_addr = ioread64(mmio + XEON_SBAR23BASE_OFFSET);
2240 dev_dbg(ndev_dev(ndev), "SBAR23 %#018llx\n", bar_addr);
2242 if (!ndev->bar4_split) {
2243 bar_addr = addr->bar4_addr64 +
2244 (b2b_bar == 4 ? ndev->b2b_off : 0);
2245 iowrite64(bar_addr, mmio + XEON_SBAR45BASE_OFFSET);
2246 bar_addr = ioread64(mmio + XEON_SBAR45BASE_OFFSET);
2247 dev_dbg(ndev_dev(ndev), "SBAR45 %#018llx\n", bar_addr);
2249 bar_addr = addr->bar4_addr32 +
2250 (b2b_bar == 4 ? ndev->b2b_off : 0);
2251 iowrite32(bar_addr, mmio + XEON_SBAR4BASE_OFFSET);
2252 bar_addr = ioread32(mmio + XEON_SBAR4BASE_OFFSET);
2253 dev_dbg(ndev_dev(ndev), "SBAR4 %#010llx\n", bar_addr);
2255 bar_addr = addr->bar5_addr32 +
2256 (b2b_bar == 5 ? ndev->b2b_off : 0);
2257 iowrite32(bar_addr, mmio + XEON_SBAR5BASE_OFFSET);
2258 bar_addr = ioread32(mmio + XEON_SBAR5BASE_OFFSET);
2259 dev_dbg(ndev_dev(ndev), "SBAR5 %#010llx\n", bar_addr);
2262 /* setup incoming bar limits == base addrs (zero length windows) */
2264 bar_addr = addr->bar2_addr64 + (b2b_bar == 2 ? ndev->b2b_off : 0);
2265 iowrite64(bar_addr, mmio + XEON_SBAR23LMT_OFFSET);
2266 bar_addr = ioread64(mmio + XEON_SBAR23LMT_OFFSET);
2267 dev_dbg(ndev_dev(ndev), "SBAR23LMT %#018llx\n", bar_addr);
2269 if (!ndev->bar4_split) {
2270 bar_addr = addr->bar4_addr64 +
2271 (b2b_bar == 4 ? ndev->b2b_off : 0);
2272 iowrite64(bar_addr, mmio + XEON_SBAR45LMT_OFFSET);
2273 bar_addr = ioread64(mmio + XEON_SBAR45LMT_OFFSET);
2274 dev_dbg(ndev_dev(ndev), "SBAR45LMT %#018llx\n", bar_addr);
2276 bar_addr = addr->bar4_addr32 +
2277 (b2b_bar == 4 ? ndev->b2b_off : 0);
2278 iowrite32(bar_addr, mmio + XEON_SBAR4LMT_OFFSET);
2279 bar_addr = ioread32(mmio + XEON_SBAR4LMT_OFFSET);
2280 dev_dbg(ndev_dev(ndev), "SBAR4LMT %#010llx\n", bar_addr);
2282 bar_addr = addr->bar5_addr32 +
2283 (b2b_bar == 5 ? ndev->b2b_off : 0);
2284 iowrite32(bar_addr, mmio + XEON_SBAR5LMT_OFFSET);
2285 bar_addr = ioread32(mmio + XEON_SBAR5LMT_OFFSET);
2286 dev_dbg(ndev_dev(ndev), "SBAR5LMT %#05llx\n", bar_addr);
2289 /* zero incoming translation addrs */
2290 iowrite64(0, mmio + XEON_SBAR23XLAT_OFFSET);
2292 if (!ndev->bar4_split) {
2293 iowrite64(0, mmio + XEON_SBAR45XLAT_OFFSET);
2295 iowrite32(0, mmio + XEON_SBAR4XLAT_OFFSET);
2296 iowrite32(0, mmio + XEON_SBAR5XLAT_OFFSET);
2299 /* zero outgoing translation limits (whole bar size windows) */
2300 iowrite64(0, mmio + XEON_PBAR23LMT_OFFSET);
2301 if (!ndev->bar4_split) {
2302 iowrite64(0, mmio + XEON_PBAR45LMT_OFFSET);
2304 iowrite32(0, mmio + XEON_PBAR4LMT_OFFSET);
2305 iowrite32(0, mmio + XEON_PBAR5LMT_OFFSET);
2308 /* set outgoing translation offsets */
2309 bar_addr = peer_addr->bar2_addr64;
2310 iowrite64(bar_addr, mmio + XEON_PBAR23XLAT_OFFSET);
2311 bar_addr = ioread64(mmio + XEON_PBAR23XLAT_OFFSET);
2312 dev_dbg(ndev_dev(ndev), "PBAR23XLAT %#018llx\n", bar_addr);
2314 if (!ndev->bar4_split) {
2315 bar_addr = peer_addr->bar4_addr64;
2316 iowrite64(bar_addr, mmio + XEON_PBAR45XLAT_OFFSET);
2317 bar_addr = ioread64(mmio + XEON_PBAR45XLAT_OFFSET);
2318 dev_dbg(ndev_dev(ndev), "PBAR45XLAT %#018llx\n", bar_addr);
2320 bar_addr = peer_addr->bar4_addr32;
2321 iowrite32(bar_addr, mmio + XEON_PBAR4XLAT_OFFSET);
2322 bar_addr = ioread32(mmio + XEON_PBAR4XLAT_OFFSET);
2323 dev_dbg(ndev_dev(ndev), "PBAR4XLAT %#010llx\n", bar_addr);
2325 bar_addr = peer_addr->bar5_addr32;
2326 iowrite32(bar_addr, mmio + XEON_PBAR5XLAT_OFFSET);
2327 bar_addr = ioread32(mmio + XEON_PBAR5XLAT_OFFSET);
2328 dev_dbg(ndev_dev(ndev), "PBAR5XLAT %#010llx\n", bar_addr);
2331 /* set the translation offset for b2b registers */
2333 bar_addr = peer_addr->bar0_addr;
2334 else if (b2b_bar == 2)
2335 bar_addr = peer_addr->bar2_addr64;
2336 else if (b2b_bar == 4 && !ndev->bar4_split)
2337 bar_addr = peer_addr->bar4_addr64;
2338 else if (b2b_bar == 4)
2339 bar_addr = peer_addr->bar4_addr32;
2340 else if (b2b_bar == 5)
2341 bar_addr = peer_addr->bar5_addr32;
2345 /* B2B_XLAT_OFFSET is 64bit, but can only take 32bit writes */
2346 dev_dbg(ndev_dev(ndev), "B2BXLAT %#018llx\n", bar_addr);
2347 iowrite32(bar_addr, mmio + XEON_B2B_XLAT_OFFSETL);
2348 iowrite32(bar_addr >> 32, mmio + XEON_B2B_XLAT_OFFSETU);
2351 /* map peer ntb mmio config space registers */
2352 ndev->peer_mmio = pci_iomap(pdev, b2b_bar,
2354 if (!ndev->peer_mmio)
2357 ndev->peer_addr = pci_resource_start(pdev, b2b_bar);
2363 static int xeon_init_ntb(struct intel_ntb_dev *ndev)
2368 if (ndev->bar4_split)
2369 ndev->mw_count = HSX_SPLIT_BAR_MW_COUNT;
2371 ndev->mw_count = XEON_MW_COUNT;
2373 ndev->spad_count = XEON_SPAD_COUNT;
2374 ndev->db_count = XEON_DB_COUNT;
2375 ndev->db_link_mask = XEON_DB_LINK_BIT;
2377 switch (ndev->ntb.topo) {
2379 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2380 dev_err(ndev_dev(ndev), "NTB Primary config disabled\n");
2384 /* enable link to allow secondary side device to appear */
2385 ntb_ctl = ioread32(ndev->self_mmio + ndev->reg->ntb_ctl);
2386 ntb_ctl &= ~NTB_CTL_DISABLE;
2387 iowrite32(ntb_ctl, ndev->self_mmio + ndev->reg->ntb_ctl);
2389 /* use half the spads for the peer */
2390 ndev->spad_count >>= 1;
2391 ndev->self_reg = &xeon_pri_reg;
2392 ndev->peer_reg = &xeon_sec_reg;
2393 ndev->xlat_reg = &xeon_sec_xlat;
2397 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2398 dev_err(ndev_dev(ndev), "NTB Secondary config disabled\n");
2401 /* use half the spads for the peer */
2402 ndev->spad_count >>= 1;
2403 ndev->self_reg = &xeon_sec_reg;
2404 ndev->peer_reg = &xeon_pri_reg;
2405 ndev->xlat_reg = &xeon_pri_xlat;
2408 case NTB_TOPO_B2B_USD:
2409 case NTB_TOPO_B2B_DSD:
2410 ndev->self_reg = &xeon_pri_reg;
2411 ndev->peer_reg = &xeon_b2b_reg;
2412 ndev->xlat_reg = &xeon_sec_xlat;
2414 if (ndev->hwerr_flags & NTB_HWERR_SDOORBELL_LOCKUP) {
2415 ndev->peer_reg = &xeon_pri_reg;
2418 ndev->b2b_idx = b2b_mw_idx + ndev->mw_count;
2420 ndev->b2b_idx = b2b_mw_idx;
2422 if (ndev->b2b_idx >= ndev->mw_count) {
2423 dev_dbg(ndev_dev(ndev),
2424 "b2b_mw_idx %d invalid for mw_count %u\n",
2425 b2b_mw_idx, ndev->mw_count);
2429 dev_dbg(ndev_dev(ndev),
2430 "setting up b2b mw idx %d means %d\n",
2431 b2b_mw_idx, ndev->b2b_idx);
2433 } else if (ndev->hwerr_flags & NTB_HWERR_B2BDOORBELL_BIT14) {
2434 dev_warn(ndev_dev(ndev), "Reduce doorbell count by 1\n");
2435 ndev->db_count -= 1;
2438 if (ndev->ntb.topo == NTB_TOPO_B2B_USD) {
2439 rc = xeon_setup_b2b_mw(ndev,
2441 &xeon_b2b_usd_addr);
2443 rc = xeon_setup_b2b_mw(ndev,
2445 &xeon_b2b_dsd_addr);
2450 /* Enable Bus Master and Memory Space on the secondary side */
2451 iowrite16(PCI_COMMAND_MEMORY | PCI_COMMAND_MASTER,
2452 ndev->self_mmio + XEON_SPCICMD_OFFSET);
2460 ndev->db_valid_mask = BIT_ULL(ndev->db_count) - 1;
2462 ndev->reg->db_iowrite(ndev->db_valid_mask,
2464 ndev->self_reg->db_mask);
2469 static int xeon_init_dev(struct intel_ntb_dev *ndev)
2471 struct pci_dev *pdev;
2475 pdev = ndev_pdev(ndev);
2477 switch (pdev->device) {
2478 /* There is a Xeon hardware errata related to writes to SDOORBELL or
2479 * B2BDOORBELL in conjunction with inbound access to NTB MMIO Space,
2480 * which may hang the system. To workaround this use the second memory
2481 * window to access the interrupt and scratch pad registers on the
2484 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
2485 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
2486 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
2487 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
2488 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
2489 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
2490 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2491 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2492 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2493 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2494 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2495 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2496 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2497 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2498 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2499 ndev->hwerr_flags |= NTB_HWERR_SDOORBELL_LOCKUP;
2503 switch (pdev->device) {
2504 /* There is a hardware errata related to accessing any register in
2505 * SB01BASE in the presence of bidirectional traffic crossing the NTB.
2507 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2508 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2509 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2510 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2511 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2512 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2513 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2514 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2515 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2516 ndev->hwerr_flags |= NTB_HWERR_SB01BASE_LOCKUP;
2520 switch (pdev->device) {
2521 /* HW Errata on bit 14 of b2bdoorbell register. Writes will not be
2522 * mirrored to the remote system. Shrink the number of bits by one,
2523 * since bit 14 is the last bit.
2525 case PCI_DEVICE_ID_INTEL_NTB_SS_JSF:
2526 case PCI_DEVICE_ID_INTEL_NTB_PS_JSF:
2527 case PCI_DEVICE_ID_INTEL_NTB_B2B_JSF:
2528 case PCI_DEVICE_ID_INTEL_NTB_SS_SNB:
2529 case PCI_DEVICE_ID_INTEL_NTB_PS_SNB:
2530 case PCI_DEVICE_ID_INTEL_NTB_B2B_SNB:
2531 case PCI_DEVICE_ID_INTEL_NTB_SS_IVT:
2532 case PCI_DEVICE_ID_INTEL_NTB_PS_IVT:
2533 case PCI_DEVICE_ID_INTEL_NTB_B2B_IVT:
2534 case PCI_DEVICE_ID_INTEL_NTB_SS_HSX:
2535 case PCI_DEVICE_ID_INTEL_NTB_PS_HSX:
2536 case PCI_DEVICE_ID_INTEL_NTB_B2B_HSX:
2537 case PCI_DEVICE_ID_INTEL_NTB_SS_BDX:
2538 case PCI_DEVICE_ID_INTEL_NTB_PS_BDX:
2539 case PCI_DEVICE_ID_INTEL_NTB_B2B_BDX:
2540 ndev->hwerr_flags |= NTB_HWERR_B2BDOORBELL_BIT14;
2544 ndev->reg = &xeon_reg;
2546 rc = pci_read_config_byte(pdev, XEON_PPD_OFFSET, &ppd);
2550 ndev->ntb.topo = xeon_ppd_topo(ndev, ppd);
2551 dev_dbg(ndev_dev(ndev), "ppd %#x topo %s\n", ppd,
2552 ntb_topo_string(ndev->ntb.topo));
2553 if (ndev->ntb.topo == NTB_TOPO_NONE)
2556 if (ndev->ntb.topo != NTB_TOPO_SEC) {
2557 ndev->bar4_split = xeon_ppd_bar4_split(ndev, ppd);
2558 dev_dbg(ndev_dev(ndev), "ppd %#x bar4_split %d\n",
2559 ppd, ndev->bar4_split);
2561 /* This is a way for transparent BAR to figure out if we are
2562 * doing split BAR or not. There is no way for the hw on the
2563 * transparent side to know and set the PPD.
2565 mem = pci_select_bars(pdev, IORESOURCE_MEM);
2566 ndev->bar4_split = hweight32(mem) ==
2567 HSX_SPLIT_BAR_MW_COUNT + 1;
2568 dev_dbg(ndev_dev(ndev), "mem %#x bar4_split %d\n",
2569 mem, ndev->bar4_split);
2572 rc = xeon_init_ntb(ndev);
2576 return xeon_init_isr(ndev);
2579 static void xeon_deinit_dev(struct intel_ntb_dev *ndev)
2581 xeon_deinit_isr(ndev);
2584 static int intel_ntb_init_pci(struct intel_ntb_dev *ndev, struct pci_dev *pdev)
2588 pci_set_drvdata(pdev, ndev);
2590 rc = pci_enable_device(pdev);
2592 goto err_pci_enable;
2594 rc = pci_request_regions(pdev, NTB_NAME);
2596 goto err_pci_regions;
2598 pci_set_master(pdev);
2600 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2602 rc = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2605 dev_warn(ndev_dev(ndev), "Cannot DMA highmem\n");
2608 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2610 rc = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
2613 dev_warn(ndev_dev(ndev), "Cannot DMA consistent highmem\n");
2616 ndev->self_mmio = pci_iomap(pdev, 0, 0);
2617 if (!ndev->self_mmio) {
2621 ndev->peer_mmio = ndev->self_mmio;
2622 ndev->peer_addr = pci_resource_start(pdev, 0);
2628 pci_clear_master(pdev);
2629 pci_release_regions(pdev);
2631 pci_disable_device(pdev);
2633 pci_set_drvdata(pdev, NULL);
2637 static void intel_ntb_deinit_pci(struct intel_ntb_dev *ndev)
2639 struct pci_dev *pdev = ndev_pdev(ndev);
2641 if (ndev->peer_mmio && ndev->peer_mmio != ndev->self_mmio)
2642 pci_iounmap(pdev, ndev->peer_mmio);
2643 pci_iounmap(pdev, ndev->self_mmio);
2645 pci_clear_master(pdev);
2646 pci_release_regions(pdev);
2647 pci_disable_device(pdev);
2648 pci_set_drvdata(pdev, NULL);
2651 static inline void ndev_init_struct(struct intel_ntb_dev *ndev,
2652 struct pci_dev *pdev)
2654 ndev->ntb.pdev = pdev;
2655 ndev->ntb.topo = NTB_TOPO_NONE;
2656 ndev->ntb.ops = &intel_ntb_ops;
2659 ndev->b2b_idx = UINT_MAX;
2661 ndev->bar4_split = 0;
2664 ndev->spad_count = 0;
2666 ndev->db_vec_count = 0;
2667 ndev->db_vec_shift = 0;
2672 ndev->db_valid_mask = 0;
2673 ndev->db_link_mask = 0;
2676 spin_lock_init(&ndev->db_mask_lock);
2679 static int intel_ntb_pci_probe(struct pci_dev *pdev,
2680 const struct pci_device_id *id)
2682 struct intel_ntb_dev *ndev;
2685 node = dev_to_node(&pdev->dev);
2687 if (pdev_is_atom(pdev)) {
2688 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2694 ndev_init_struct(ndev, pdev);
2696 rc = intel_ntb_init_pci(ndev, pdev);
2700 rc = atom_init_dev(ndev);
2704 } else if (pdev_is_xeon(pdev)) {
2705 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2711 ndev_init_struct(ndev, pdev);
2713 rc = intel_ntb_init_pci(ndev, pdev);
2717 rc = xeon_init_dev(ndev);
2721 } else if (pdev_is_skx_xeon(pdev)) {
2722 ndev = kzalloc_node(sizeof(*ndev), GFP_KERNEL, node);
2728 ndev_init_struct(ndev, pdev);
2729 ndev->ntb.ops = &intel_ntb3_ops;
2731 rc = intel_ntb_init_pci(ndev, pdev);
2735 rc = skx_init_dev(ndev);
2744 ndev_reset_unsafe_flags(ndev);
2746 ndev->reg->poll_link(ndev);
2748 ndev_init_debugfs(ndev);
2750 rc = ntb_register_device(&ndev->ntb);
2754 dev_info(&pdev->dev, "NTB device registered.\n");
2759 ndev_deinit_debugfs(ndev);
2760 if (pdev_is_atom(pdev))
2761 atom_deinit_dev(ndev);
2762 else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2763 xeon_deinit_dev(ndev);
2765 intel_ntb_deinit_pci(ndev);
2772 static void intel_ntb_pci_remove(struct pci_dev *pdev)
2774 struct intel_ntb_dev *ndev = pci_get_drvdata(pdev);
2776 ntb_unregister_device(&ndev->ntb);
2777 ndev_deinit_debugfs(ndev);
2778 if (pdev_is_atom(pdev))
2779 atom_deinit_dev(ndev);
2780 else if (pdev_is_xeon(pdev) || pdev_is_skx_xeon(pdev))
2781 xeon_deinit_dev(ndev);
2782 intel_ntb_deinit_pci(ndev);
2786 static const struct intel_ntb_reg atom_reg = {
2787 .poll_link = atom_poll_link,
2788 .link_is_up = atom_link_is_up,
2789 .db_ioread = atom_db_ioread,
2790 .db_iowrite = atom_db_iowrite,
2791 .db_size = sizeof(u64),
2792 .ntb_ctl = ATOM_NTBCNTL_OFFSET,
2796 static const struct intel_ntb_alt_reg atom_pri_reg = {
2797 .db_bell = ATOM_PDOORBELL_OFFSET,
2798 .db_mask = ATOM_PDBMSK_OFFSET,
2799 .spad = ATOM_SPAD_OFFSET,
2802 static const struct intel_ntb_alt_reg atom_b2b_reg = {
2803 .db_bell = ATOM_B2B_DOORBELL_OFFSET,
2804 .spad = ATOM_B2B_SPAD_OFFSET,
2807 static const struct intel_ntb_xlat_reg atom_sec_xlat = {
2808 /* FIXME : .bar0_base = ATOM_SBAR0BASE_OFFSET, */
2809 /* FIXME : .bar2_limit = ATOM_SBAR2LMT_OFFSET, */
2810 .bar2_xlat = ATOM_SBAR2XLAT_OFFSET,
2813 static const struct intel_ntb_reg xeon_reg = {
2814 .poll_link = xeon_poll_link,
2815 .link_is_up = xeon_link_is_up,
2816 .db_ioread = xeon_db_ioread,
2817 .db_iowrite = xeon_db_iowrite,
2818 .db_size = sizeof(u32),
2819 .ntb_ctl = XEON_NTBCNTL_OFFSET,
2820 .mw_bar = {2, 4, 5},
2823 static const struct intel_ntb_alt_reg xeon_pri_reg = {
2824 .db_bell = XEON_PDOORBELL_OFFSET,
2825 .db_mask = XEON_PDBMSK_OFFSET,
2826 .spad = XEON_SPAD_OFFSET,
2829 static const struct intel_ntb_alt_reg xeon_sec_reg = {
2830 .db_bell = XEON_SDOORBELL_OFFSET,
2831 .db_mask = XEON_SDBMSK_OFFSET,
2832 /* second half of the scratchpads */
2833 .spad = XEON_SPAD_OFFSET + (XEON_SPAD_COUNT << 1),
2836 static const struct intel_ntb_alt_reg xeon_b2b_reg = {
2837 .db_bell = XEON_B2B_DOORBELL_OFFSET,
2838 .spad = XEON_B2B_SPAD_OFFSET,
2841 static const struct intel_ntb_xlat_reg xeon_pri_xlat = {
2842 /* Note: no primary .bar0_base visible to the secondary side.
2844 * The secondary side cannot get the base address stored in primary
2845 * bars. The base address is necessary to set the limit register to
2846 * any value other than zero, or unlimited.
2848 * WITHOUT THE BASE ADDRESS, THE SECONDARY SIDE CANNOT DISABLE the
2849 * window by setting the limit equal to base, nor can it limit the size
2850 * of the memory window by setting the limit to base + size.
2852 .bar2_limit = XEON_PBAR23LMT_OFFSET,
2853 .bar2_xlat = XEON_PBAR23XLAT_OFFSET,
2856 static const struct intel_ntb_xlat_reg xeon_sec_xlat = {
2857 .bar0_base = XEON_SBAR0BASE_OFFSET,
2858 .bar2_limit = XEON_SBAR23LMT_OFFSET,
2859 .bar2_xlat = XEON_SBAR23XLAT_OFFSET,
2862 static struct intel_b2b_addr xeon_b2b_usd_addr = {
2863 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
2864 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
2865 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
2866 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
2869 static struct intel_b2b_addr xeon_b2b_dsd_addr = {
2870 .bar2_addr64 = XEON_B2B_BAR2_ADDR64,
2871 .bar4_addr64 = XEON_B2B_BAR4_ADDR64,
2872 .bar4_addr32 = XEON_B2B_BAR4_ADDR32,
2873 .bar5_addr32 = XEON_B2B_BAR5_ADDR32,
2876 static const struct intel_ntb_reg skx_reg = {
2877 .poll_link = skx_poll_link,
2878 .link_is_up = xeon_link_is_up,
2879 .db_ioread = skx_db_ioread,
2880 .db_iowrite = skx_db_iowrite,
2881 .db_size = sizeof(u64),
2882 .ntb_ctl = SKX_NTBCNTL_OFFSET,
2886 static const struct intel_ntb_alt_reg skx_pri_reg = {
2887 .db_bell = SKX_EM_DOORBELL_OFFSET,
2888 .db_clear = SKX_IM_INT_STATUS_OFFSET,
2889 .db_mask = SKX_IM_INT_DISABLE_OFFSET,
2890 .spad = SKX_IM_SPAD_OFFSET,
2893 static const struct intel_ntb_alt_reg skx_b2b_reg = {
2894 .db_bell = SKX_IM_DOORBELL_OFFSET,
2895 .db_clear = SKX_EM_INT_STATUS_OFFSET,
2896 .db_mask = SKX_EM_INT_DISABLE_OFFSET,
2897 .spad = SKX_B2B_SPAD_OFFSET,
2900 static const struct intel_ntb_xlat_reg skx_sec_xlat = {
2901 /* .bar0_base = SKX_EMBAR0_OFFSET, */
2902 .bar2_limit = SKX_IMBAR1XLMT_OFFSET,
2903 .bar2_xlat = SKX_IMBAR1XBASE_OFFSET,
2906 /* operations for primary side of local ntb */
2907 static const struct ntb_dev_ops intel_ntb_ops = {
2908 .mw_count = intel_ntb_mw_count,
2909 .mw_get_range = intel_ntb_mw_get_range,
2910 .mw_set_trans = intel_ntb_mw_set_trans,
2911 .link_is_up = intel_ntb_link_is_up,
2912 .link_enable = intel_ntb_link_enable,
2913 .link_disable = intel_ntb_link_disable,
2914 .db_is_unsafe = intel_ntb_db_is_unsafe,
2915 .db_valid_mask = intel_ntb_db_valid_mask,
2916 .db_vector_count = intel_ntb_db_vector_count,
2917 .db_vector_mask = intel_ntb_db_vector_mask,
2918 .db_read = intel_ntb_db_read,
2919 .db_clear = intel_ntb_db_clear,
2920 .db_set_mask = intel_ntb_db_set_mask,
2921 .db_clear_mask = intel_ntb_db_clear_mask,
2922 .peer_db_addr = intel_ntb_peer_db_addr,
2923 .peer_db_set = intel_ntb_peer_db_set,
2924 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2925 .spad_count = intel_ntb_spad_count,
2926 .spad_read = intel_ntb_spad_read,
2927 .spad_write = intel_ntb_spad_write,
2928 .peer_spad_addr = intel_ntb_peer_spad_addr,
2929 .peer_spad_read = intel_ntb_peer_spad_read,
2930 .peer_spad_write = intel_ntb_peer_spad_write,
2933 static const struct ntb_dev_ops intel_ntb3_ops = {
2934 .mw_count = intel_ntb_mw_count,
2935 .mw_get_range = intel_ntb_mw_get_range,
2936 .mw_set_trans = intel_ntb3_mw_set_trans,
2937 .link_is_up = intel_ntb_link_is_up,
2938 .link_enable = intel_ntb3_link_enable,
2939 .link_disable = intel_ntb_link_disable,
2940 .db_valid_mask = intel_ntb_db_valid_mask,
2941 .db_vector_count = intel_ntb_db_vector_count,
2942 .db_vector_mask = intel_ntb_db_vector_mask,
2943 .db_read = intel_ntb3_db_read,
2944 .db_clear = intel_ntb3_db_clear,
2945 .db_set_mask = intel_ntb_db_set_mask,
2946 .db_clear_mask = intel_ntb_db_clear_mask,
2947 .peer_db_addr = intel_ntb_peer_db_addr,
2948 .peer_db_set = intel_ntb3_peer_db_set,
2949 .spad_is_unsafe = intel_ntb_spad_is_unsafe,
2950 .spad_count = intel_ntb_spad_count,
2951 .spad_read = intel_ntb_spad_read,
2952 .spad_write = intel_ntb_spad_write,
2953 .peer_spad_addr = intel_ntb_peer_spad_addr,
2954 .peer_spad_read = intel_ntb_peer_spad_read,
2955 .peer_spad_write = intel_ntb_peer_spad_write,
2958 static const struct file_operations intel_ntb_debugfs_info = {
2959 .owner = THIS_MODULE,
2960 .open = simple_open,
2961 .read = ndev_debugfs_read,
2964 static const struct pci_device_id intel_ntb_pci_tbl[] = {
2965 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BWD)},
2966 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_JSF)},
2967 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SNB)},
2968 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_IVT)},
2969 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_HSX)},
2970 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_BDX)},
2971 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_JSF)},
2972 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_SNB)},
2973 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_IVT)},
2974 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_HSX)},
2975 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_PS_BDX)},
2976 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_JSF)},
2977 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_SNB)},
2978 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_IVT)},
2979 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_HSX)},
2980 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_SS_BDX)},
2981 {PCI_VDEVICE(INTEL, PCI_DEVICE_ID_INTEL_NTB_B2B_SKX)},
2984 MODULE_DEVICE_TABLE(pci, intel_ntb_pci_tbl);
2986 static struct pci_driver intel_ntb_pci_driver = {
2987 .name = KBUILD_MODNAME,
2988 .id_table = intel_ntb_pci_tbl,
2989 .probe = intel_ntb_pci_probe,
2990 .remove = intel_ntb_pci_remove,
2993 static int __init intel_ntb_pci_driver_init(void)
2995 pr_info("%s %s\n", NTB_DESC, NTB_VER);
2997 if (debugfs_initialized())
2998 debugfs_dir = debugfs_create_dir(KBUILD_MODNAME, NULL);
3000 return pci_register_driver(&intel_ntb_pci_driver);
3002 module_init(intel_ntb_pci_driver_init);
3004 static void __exit intel_ntb_pci_driver_exit(void)
3006 pci_unregister_driver(&intel_ntb_pci_driver);
3008 debugfs_remove_recursive(debugfs_dir);
3010 module_exit(intel_ntb_pci_driver_exit);