2 * This file is provided under a GPLv2 license. When using or
3 * redistributing this file, you may do so under that license.
7 * Copyright (C) 2016-2018 T-Platforms JSC All Rights Reserved.
9 * This program is free software; you can redistribute it and/or modify it
10 * under the terms and conditions of the GNU General Public License,
11 * version 2, as published by the Free Software Foundation.
13 * This program is distributed in the hope that it will be useful, but
14 * WITHOUT ANY WARRANTY; without even the implied warranty of
15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
16 * Public License for more details.
18 * You should have received a copy of the GNU General Public License along
19 * with this program; if not, one can be found http://www.gnu.org/licenses/.
21 * The full GNU General Public License is included in this distribution in
22 * the file called "COPYING".
24 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
25 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
26 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
27 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
28 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
29 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
30 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
31 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
32 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
33 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
34 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 * IDT PCIe-switch NTB Linux driver
38 * Contact Information:
39 * Serge Semin <fancer.lancer@gmail.com>, <Sergey.Semin@t-platforms.ru>
42 #include <linux/stddef.h>
43 #include <linux/types.h>
44 #include <linux/kernel.h>
45 #include <linux/bitops.h>
46 #include <linux/sizes.h>
47 #include <linux/module.h>
48 #include <linux/moduleparam.h>
49 #include <linux/init.h>
50 #include <linux/interrupt.h>
51 #include <linux/spinlock.h>
52 #include <linux/mutex.h>
53 #include <linux/pci.h>
54 #include <linux/aer.h>
55 #include <linux/slab.h>
56 #include <linux/list.h>
57 #include <linux/debugfs.h>
58 #include <linux/hwmon.h>
59 #include <linux/hwmon-sysfs.h>
60 #include <linux/ntb.h>
62 #include "ntb_hw_idt.h"
64 #define NTB_NAME "ntb_hw_idt"
65 #define NTB_DESC "IDT PCI-E Non-Transparent Bridge Driver"
67 #define NTB_IRQNAME "ntb_irq_idt"
69 MODULE_DESCRIPTION(NTB_DESC);
70 MODULE_VERSION(NTB_VER);
71 MODULE_LICENSE("GPL v2");
72 MODULE_AUTHOR("T-platforms");
75 * NT Endpoint registers table simplifying a loop access to the functionally
78 static const struct idt_ntb_regs ntdata_tbl = {
79 { {IDT_NT_BARSETUP0, IDT_NT_BARLIMIT0,
80 IDT_NT_BARLTBASE0, IDT_NT_BARUTBASE0},
81 {IDT_NT_BARSETUP1, IDT_NT_BARLIMIT1,
82 IDT_NT_BARLTBASE1, IDT_NT_BARUTBASE1},
83 {IDT_NT_BARSETUP2, IDT_NT_BARLIMIT2,
84 IDT_NT_BARLTBASE2, IDT_NT_BARUTBASE2},
85 {IDT_NT_BARSETUP3, IDT_NT_BARLIMIT3,
86 IDT_NT_BARLTBASE3, IDT_NT_BARUTBASE3},
87 {IDT_NT_BARSETUP4, IDT_NT_BARLIMIT4,
88 IDT_NT_BARLTBASE4, IDT_NT_BARUTBASE4},
89 {IDT_NT_BARSETUP5, IDT_NT_BARLIMIT5,
90 IDT_NT_BARLTBASE5, IDT_NT_BARUTBASE5} },
91 { {IDT_NT_INMSG0, IDT_NT_OUTMSG0, IDT_NT_INMSGSRC0},
92 {IDT_NT_INMSG1, IDT_NT_OUTMSG1, IDT_NT_INMSGSRC1},
93 {IDT_NT_INMSG2, IDT_NT_OUTMSG2, IDT_NT_INMSGSRC2},
94 {IDT_NT_INMSG3, IDT_NT_OUTMSG3, IDT_NT_INMSGSRC3} }
98 * NT Endpoint ports data table with the corresponding pcie command, link
99 * status, control and BAR-related registers
101 static const struct idt_ntb_port portdata_tbl[IDT_MAX_NR_PORTS] = {
102 /*0*/ { IDT_SW_NTP0_PCIECMDSTS, IDT_SW_NTP0_PCIELCTLSTS,
104 IDT_SW_SWPORT0CTL, IDT_SW_SWPORT0STS,
105 { {IDT_SW_NTP0_BARSETUP0, IDT_SW_NTP0_BARLIMIT0,
106 IDT_SW_NTP0_BARLTBASE0, IDT_SW_NTP0_BARUTBASE0},
107 {IDT_SW_NTP0_BARSETUP1, IDT_SW_NTP0_BARLIMIT1,
108 IDT_SW_NTP0_BARLTBASE1, IDT_SW_NTP0_BARUTBASE1},
109 {IDT_SW_NTP0_BARSETUP2, IDT_SW_NTP0_BARLIMIT2,
110 IDT_SW_NTP0_BARLTBASE2, IDT_SW_NTP0_BARUTBASE2},
111 {IDT_SW_NTP0_BARSETUP3, IDT_SW_NTP0_BARLIMIT3,
112 IDT_SW_NTP0_BARLTBASE3, IDT_SW_NTP0_BARUTBASE3},
113 {IDT_SW_NTP0_BARSETUP4, IDT_SW_NTP0_BARLIMIT4,
114 IDT_SW_NTP0_BARLTBASE4, IDT_SW_NTP0_BARUTBASE4},
115 {IDT_SW_NTP0_BARSETUP5, IDT_SW_NTP0_BARLIMIT5,
116 IDT_SW_NTP0_BARLTBASE5, IDT_SW_NTP0_BARUTBASE5} } },
118 /*2*/ { IDT_SW_NTP2_PCIECMDSTS, IDT_SW_NTP2_PCIELCTLSTS,
120 IDT_SW_SWPORT2CTL, IDT_SW_SWPORT2STS,
121 { {IDT_SW_NTP2_BARSETUP0, IDT_SW_NTP2_BARLIMIT0,
122 IDT_SW_NTP2_BARLTBASE0, IDT_SW_NTP2_BARUTBASE0},
123 {IDT_SW_NTP2_BARSETUP1, IDT_SW_NTP2_BARLIMIT1,
124 IDT_SW_NTP2_BARLTBASE1, IDT_SW_NTP2_BARUTBASE1},
125 {IDT_SW_NTP2_BARSETUP2, IDT_SW_NTP2_BARLIMIT2,
126 IDT_SW_NTP2_BARLTBASE2, IDT_SW_NTP2_BARUTBASE2},
127 {IDT_SW_NTP2_BARSETUP3, IDT_SW_NTP2_BARLIMIT3,
128 IDT_SW_NTP2_BARLTBASE3, IDT_SW_NTP2_BARUTBASE3},
129 {IDT_SW_NTP2_BARSETUP4, IDT_SW_NTP2_BARLIMIT4,
130 IDT_SW_NTP2_BARLTBASE4, IDT_SW_NTP2_BARUTBASE4},
131 {IDT_SW_NTP2_BARSETUP5, IDT_SW_NTP2_BARLIMIT5,
132 IDT_SW_NTP2_BARLTBASE5, IDT_SW_NTP2_BARUTBASE5} } },
134 /*4*/ { IDT_SW_NTP4_PCIECMDSTS, IDT_SW_NTP4_PCIELCTLSTS,
136 IDT_SW_SWPORT4CTL, IDT_SW_SWPORT4STS,
137 { {IDT_SW_NTP4_BARSETUP0, IDT_SW_NTP4_BARLIMIT0,
138 IDT_SW_NTP4_BARLTBASE0, IDT_SW_NTP4_BARUTBASE0},
139 {IDT_SW_NTP4_BARSETUP1, IDT_SW_NTP4_BARLIMIT1,
140 IDT_SW_NTP4_BARLTBASE1, IDT_SW_NTP4_BARUTBASE1},
141 {IDT_SW_NTP4_BARSETUP2, IDT_SW_NTP4_BARLIMIT2,
142 IDT_SW_NTP4_BARLTBASE2, IDT_SW_NTP4_BARUTBASE2},
143 {IDT_SW_NTP4_BARSETUP3, IDT_SW_NTP4_BARLIMIT3,
144 IDT_SW_NTP4_BARLTBASE3, IDT_SW_NTP4_BARUTBASE3},
145 {IDT_SW_NTP4_BARSETUP4, IDT_SW_NTP4_BARLIMIT4,
146 IDT_SW_NTP4_BARLTBASE4, IDT_SW_NTP4_BARUTBASE4},
147 {IDT_SW_NTP4_BARSETUP5, IDT_SW_NTP4_BARLIMIT5,
148 IDT_SW_NTP4_BARLTBASE5, IDT_SW_NTP4_BARUTBASE5} } },
150 /*6*/ { IDT_SW_NTP6_PCIECMDSTS, IDT_SW_NTP6_PCIELCTLSTS,
152 IDT_SW_SWPORT6CTL, IDT_SW_SWPORT6STS,
153 { {IDT_SW_NTP6_BARSETUP0, IDT_SW_NTP6_BARLIMIT0,
154 IDT_SW_NTP6_BARLTBASE0, IDT_SW_NTP6_BARUTBASE0},
155 {IDT_SW_NTP6_BARSETUP1, IDT_SW_NTP6_BARLIMIT1,
156 IDT_SW_NTP6_BARLTBASE1, IDT_SW_NTP6_BARUTBASE1},
157 {IDT_SW_NTP6_BARSETUP2, IDT_SW_NTP6_BARLIMIT2,
158 IDT_SW_NTP6_BARLTBASE2, IDT_SW_NTP6_BARUTBASE2},
159 {IDT_SW_NTP6_BARSETUP3, IDT_SW_NTP6_BARLIMIT3,
160 IDT_SW_NTP6_BARLTBASE3, IDT_SW_NTP6_BARUTBASE3},
161 {IDT_SW_NTP6_BARSETUP4, IDT_SW_NTP6_BARLIMIT4,
162 IDT_SW_NTP6_BARLTBASE4, IDT_SW_NTP6_BARUTBASE4},
163 {IDT_SW_NTP6_BARSETUP5, IDT_SW_NTP6_BARLIMIT5,
164 IDT_SW_NTP6_BARLTBASE5, IDT_SW_NTP6_BARUTBASE5} } },
166 /*8*/ { IDT_SW_NTP8_PCIECMDSTS, IDT_SW_NTP8_PCIELCTLSTS,
168 IDT_SW_SWPORT8CTL, IDT_SW_SWPORT8STS,
169 { {IDT_SW_NTP8_BARSETUP0, IDT_SW_NTP8_BARLIMIT0,
170 IDT_SW_NTP8_BARLTBASE0, IDT_SW_NTP8_BARUTBASE0},
171 {IDT_SW_NTP8_BARSETUP1, IDT_SW_NTP8_BARLIMIT1,
172 IDT_SW_NTP8_BARLTBASE1, IDT_SW_NTP8_BARUTBASE1},
173 {IDT_SW_NTP8_BARSETUP2, IDT_SW_NTP8_BARLIMIT2,
174 IDT_SW_NTP8_BARLTBASE2, IDT_SW_NTP8_BARUTBASE2},
175 {IDT_SW_NTP8_BARSETUP3, IDT_SW_NTP8_BARLIMIT3,
176 IDT_SW_NTP8_BARLTBASE3, IDT_SW_NTP8_BARUTBASE3},
177 {IDT_SW_NTP8_BARSETUP4, IDT_SW_NTP8_BARLIMIT4,
178 IDT_SW_NTP8_BARLTBASE4, IDT_SW_NTP8_BARUTBASE4},
179 {IDT_SW_NTP8_BARSETUP5, IDT_SW_NTP8_BARLIMIT5,
180 IDT_SW_NTP8_BARLTBASE5, IDT_SW_NTP8_BARUTBASE5} } },
184 /*12*/ { IDT_SW_NTP12_PCIECMDSTS, IDT_SW_NTP12_PCIELCTLSTS,
186 IDT_SW_SWPORT12CTL, IDT_SW_SWPORT12STS,
187 { {IDT_SW_NTP12_BARSETUP0, IDT_SW_NTP12_BARLIMIT0,
188 IDT_SW_NTP12_BARLTBASE0, IDT_SW_NTP12_BARUTBASE0},
189 {IDT_SW_NTP12_BARSETUP1, IDT_SW_NTP12_BARLIMIT1,
190 IDT_SW_NTP12_BARLTBASE1, IDT_SW_NTP12_BARUTBASE1},
191 {IDT_SW_NTP12_BARSETUP2, IDT_SW_NTP12_BARLIMIT2,
192 IDT_SW_NTP12_BARLTBASE2, IDT_SW_NTP12_BARUTBASE2},
193 {IDT_SW_NTP12_BARSETUP3, IDT_SW_NTP12_BARLIMIT3,
194 IDT_SW_NTP12_BARLTBASE3, IDT_SW_NTP12_BARUTBASE3},
195 {IDT_SW_NTP12_BARSETUP4, IDT_SW_NTP12_BARLIMIT4,
196 IDT_SW_NTP12_BARLTBASE4, IDT_SW_NTP12_BARUTBASE4},
197 {IDT_SW_NTP12_BARSETUP5, IDT_SW_NTP12_BARLIMIT5,
198 IDT_SW_NTP12_BARLTBASE5, IDT_SW_NTP12_BARUTBASE5} } },
202 /*16*/ { IDT_SW_NTP16_PCIECMDSTS, IDT_SW_NTP16_PCIELCTLSTS,
204 IDT_SW_SWPORT16CTL, IDT_SW_SWPORT16STS,
205 { {IDT_SW_NTP16_BARSETUP0, IDT_SW_NTP16_BARLIMIT0,
206 IDT_SW_NTP16_BARLTBASE0, IDT_SW_NTP16_BARUTBASE0},
207 {IDT_SW_NTP16_BARSETUP1, IDT_SW_NTP16_BARLIMIT1,
208 IDT_SW_NTP16_BARLTBASE1, IDT_SW_NTP16_BARUTBASE1},
209 {IDT_SW_NTP16_BARSETUP2, IDT_SW_NTP16_BARLIMIT2,
210 IDT_SW_NTP16_BARLTBASE2, IDT_SW_NTP16_BARUTBASE2},
211 {IDT_SW_NTP16_BARSETUP3, IDT_SW_NTP16_BARLIMIT3,
212 IDT_SW_NTP16_BARLTBASE3, IDT_SW_NTP16_BARUTBASE3},
213 {IDT_SW_NTP16_BARSETUP4, IDT_SW_NTP16_BARLIMIT4,
214 IDT_SW_NTP16_BARLTBASE4, IDT_SW_NTP16_BARUTBASE4},
215 {IDT_SW_NTP16_BARSETUP5, IDT_SW_NTP16_BARLIMIT5,
216 IDT_SW_NTP16_BARLTBASE5, IDT_SW_NTP16_BARUTBASE5} } },
220 /*20*/ { IDT_SW_NTP20_PCIECMDSTS, IDT_SW_NTP20_PCIELCTLSTS,
222 IDT_SW_SWPORT20CTL, IDT_SW_SWPORT20STS,
223 { {IDT_SW_NTP20_BARSETUP0, IDT_SW_NTP20_BARLIMIT0,
224 IDT_SW_NTP20_BARLTBASE0, IDT_SW_NTP20_BARUTBASE0},
225 {IDT_SW_NTP20_BARSETUP1, IDT_SW_NTP20_BARLIMIT1,
226 IDT_SW_NTP20_BARLTBASE1, IDT_SW_NTP20_BARUTBASE1},
227 {IDT_SW_NTP20_BARSETUP2, IDT_SW_NTP20_BARLIMIT2,
228 IDT_SW_NTP20_BARLTBASE2, IDT_SW_NTP20_BARUTBASE2},
229 {IDT_SW_NTP20_BARSETUP3, IDT_SW_NTP20_BARLIMIT3,
230 IDT_SW_NTP20_BARLTBASE3, IDT_SW_NTP20_BARUTBASE3},
231 {IDT_SW_NTP20_BARSETUP4, IDT_SW_NTP20_BARLIMIT4,
232 IDT_SW_NTP20_BARLTBASE4, IDT_SW_NTP20_BARUTBASE4},
233 {IDT_SW_NTP20_BARSETUP5, IDT_SW_NTP20_BARLIMIT5,
234 IDT_SW_NTP20_BARLTBASE5, IDT_SW_NTP20_BARUTBASE5} } },
241 * IDT PCIe-switch partitions table with the corresponding control, status
242 * and messages control registers
244 static const struct idt_ntb_part partdata_tbl[IDT_MAX_NR_PARTS] = {
245 /*0*/ { IDT_SW_SWPART0CTL, IDT_SW_SWPART0STS,
246 {IDT_SW_SWP0MSGCTL0, IDT_SW_SWP0MSGCTL1,
247 IDT_SW_SWP0MSGCTL2, IDT_SW_SWP0MSGCTL3} },
248 /*1*/ { IDT_SW_SWPART1CTL, IDT_SW_SWPART1STS,
249 {IDT_SW_SWP1MSGCTL0, IDT_SW_SWP1MSGCTL1,
250 IDT_SW_SWP1MSGCTL2, IDT_SW_SWP1MSGCTL3} },
251 /*2*/ { IDT_SW_SWPART2CTL, IDT_SW_SWPART2STS,
252 {IDT_SW_SWP2MSGCTL0, IDT_SW_SWP2MSGCTL1,
253 IDT_SW_SWP2MSGCTL2, IDT_SW_SWP2MSGCTL3} },
254 /*3*/ { IDT_SW_SWPART3CTL, IDT_SW_SWPART3STS,
255 {IDT_SW_SWP3MSGCTL0, IDT_SW_SWP3MSGCTL1,
256 IDT_SW_SWP3MSGCTL2, IDT_SW_SWP3MSGCTL3} },
257 /*4*/ { IDT_SW_SWPART4CTL, IDT_SW_SWPART4STS,
258 {IDT_SW_SWP4MSGCTL0, IDT_SW_SWP4MSGCTL1,
259 IDT_SW_SWP4MSGCTL2, IDT_SW_SWP4MSGCTL3} },
260 /*5*/ { IDT_SW_SWPART5CTL, IDT_SW_SWPART5STS,
261 {IDT_SW_SWP5MSGCTL0, IDT_SW_SWP5MSGCTL1,
262 IDT_SW_SWP5MSGCTL2, IDT_SW_SWP5MSGCTL3} },
263 /*6*/ { IDT_SW_SWPART6CTL, IDT_SW_SWPART6STS,
264 {IDT_SW_SWP6MSGCTL0, IDT_SW_SWP6MSGCTL1,
265 IDT_SW_SWP6MSGCTL2, IDT_SW_SWP6MSGCTL3} },
266 /*7*/ { IDT_SW_SWPART7CTL, IDT_SW_SWPART7STS,
267 {IDT_SW_SWP7MSGCTL0, IDT_SW_SWP7MSGCTL1,
268 IDT_SW_SWP7MSGCTL2, IDT_SW_SWP7MSGCTL3} }
272 * DebugFS directory to place the driver debug file
274 static struct dentry *dbgfs_topdir;
276 /*=============================================================================
277 * 1. IDT PCIe-switch registers IO-functions
279 * Beside ordinary configuration space registers IDT PCIe-switch expose
280 * global configuration registers, which are used to determine state of other
281 * device ports as well as being notified of some switch-related events.
282 * Additionally all the configuration space registers of all the IDT
283 * PCIe-switch functions are mapped to the Global Address space, so each
284 * function can determine a configuration of any other PCI-function.
285 * Functions declared in this chapter are created to encapsulate access
286 * to configuration and global registers, so the driver code just need to
287 * provide IDT NTB hardware descriptor and a register address.
288 *=============================================================================
292 * idt_nt_write() - PCI configuration space registers write method
293 * @ndev: IDT NTB hardware driver descriptor
294 * @reg: Register to write data to
295 * @data: Value to write to the register
297 * IDT PCIe-switch registers are all Little endian.
299 static void idt_nt_write(struct idt_ntb_dev *ndev,
300 const unsigned int reg, const u32 data)
303 * It's obvious bug to request a register exceeding the maximum possible
304 * value as well as to have it unaligned.
306 if (WARN_ON(reg > IDT_REG_PCI_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
309 /* Just write the value to the specified register */
310 iowrite32(data, ndev->cfgspc + (ptrdiff_t)reg);
314 * idt_nt_read() - PCI configuration space registers read method
315 * @ndev: IDT NTB hardware driver descriptor
316 * @reg: Register to write data to
318 * IDT PCIe-switch Global configuration registers are all Little endian.
320 * Return: register value
322 static u32 idt_nt_read(struct idt_ntb_dev *ndev, const unsigned int reg)
325 * It's obvious bug to request a register exceeding the maximum possible
326 * value as well as to have it unaligned.
328 if (WARN_ON(reg > IDT_REG_PCI_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
331 /* Just read the value from the specified register */
332 return ioread32(ndev->cfgspc + (ptrdiff_t)reg);
336 * idt_sw_write() - Global registers write method
337 * @ndev: IDT NTB hardware driver descriptor
338 * @reg: Register to write data to
339 * @data: Value to write to the register
341 * IDT PCIe-switch Global configuration registers are all Little endian.
343 static void idt_sw_write(struct idt_ntb_dev *ndev,
344 const unsigned int reg, const u32 data)
346 unsigned long irqflags;
349 * It's obvious bug to request a register exceeding the maximum possible
350 * value as well as to have it unaligned.
352 if (WARN_ON(reg > IDT_REG_SW_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
355 /* Lock GASA registers operations */
356 spin_lock_irqsave(&ndev->gasa_lock, irqflags);
357 /* Set the global register address */
358 iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR);
359 /* Put the new value of the register */
360 iowrite32(data, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA);
361 /* Make sure the PCIe transactions are executed */
363 /* Unlock GASA registers operations */
364 spin_unlock_irqrestore(&ndev->gasa_lock, irqflags);
368 * idt_sw_read() - Global registers read method
369 * @ndev: IDT NTB hardware driver descriptor
370 * @reg: Register to write data to
372 * IDT PCIe-switch Global configuration registers are all Little endian.
374 * Return: register value
376 static u32 idt_sw_read(struct idt_ntb_dev *ndev, const unsigned int reg)
378 unsigned long irqflags;
382 * It's obvious bug to request a register exceeding the maximum possible
383 * value as well as to have it unaligned.
385 if (WARN_ON(reg > IDT_REG_SW_MAX || !IS_ALIGNED(reg, IDT_REG_ALIGN)))
388 /* Lock GASA registers operations */
389 spin_lock_irqsave(&ndev->gasa_lock, irqflags);
390 /* Set the global register address */
391 iowrite32((u32)reg, ndev->cfgspc + (ptrdiff_t)IDT_NT_GASAADDR);
392 /* Get the data of the register (read ops acts as MMIO barrier) */
393 data = ioread32(ndev->cfgspc + (ptrdiff_t)IDT_NT_GASADATA);
394 /* Unlock GASA registers operations */
395 spin_unlock_irqrestore(&ndev->gasa_lock, irqflags);
401 * idt_reg_set_bits() - set bits of a passed register
402 * @ndev: IDT NTB hardware driver descriptor
403 * @reg: Register to change bits of
404 * @reg_lock: Register access spin lock
405 * @valid_mask: Mask of valid bits
406 * @set_bits: Bitmask to set
408 * Helper method to check whether a passed bitfield is valid and set
409 * corresponding bits of a register.
411 * WARNING! Make sure the passed register isn't accessed over plane
412 * idt_nt_write() method (read method is ok to be used concurrently).
414 * Return: zero on success, negative error on invalid bitmask.
416 static inline int idt_reg_set_bits(struct idt_ntb_dev *ndev, unsigned int reg,
417 spinlock_t *reg_lock,
418 u64 valid_mask, u64 set_bits)
420 unsigned long irqflags;
423 if (set_bits & ~(u64)valid_mask)
426 /* Lock access to the register unless the change is written back */
427 spin_lock_irqsave(reg_lock, irqflags);
428 data = idt_nt_read(ndev, reg) | (u32)set_bits;
429 idt_nt_write(ndev, reg, data);
430 /* Unlock the register */
431 spin_unlock_irqrestore(reg_lock, irqflags);
437 * idt_reg_clear_bits() - clear bits of a passed register
438 * @ndev: IDT NTB hardware driver descriptor
439 * @reg: Register to change bits of
440 * @reg_lock: Register access spin lock
441 * @set_bits: Bitmask to clear
443 * Helper method to check whether a passed bitfield is valid and clear
444 * corresponding bits of a register.
446 * NOTE! Invalid bits are always considered cleared so it's not an error
447 * to clear them over.
449 * WARNING! Make sure the passed register isn't accessed over plane
450 * idt_nt_write() method (read method is ok to use concurrently).
452 static inline void idt_reg_clear_bits(struct idt_ntb_dev *ndev,
453 unsigned int reg, spinlock_t *reg_lock,
456 unsigned long irqflags;
459 /* Lock access to the register unless the change is written back */
460 spin_lock_irqsave(reg_lock, irqflags);
461 data = idt_nt_read(ndev, reg) & ~(u32)clear_bits;
462 idt_nt_write(ndev, reg, data);
463 /* Unlock the register */
464 spin_unlock_irqrestore(reg_lock, irqflags);
467 /*===========================================================================
468 * 2. Ports operations
470 * IDT PCIe-switches can have from 3 up to 8 ports with possible
471 * NT-functions enabled. So all the possible ports need to be scanned looking
472 * for NTB activated. NTB API will have enumerated only the ports with NTB.
473 *===========================================================================
477 * idt_scan_ports() - scan IDT PCIe-switch ports collecting info in the tables
478 * @ndev: Pointer to the PCI device descriptor
480 * Return: zero on success, otherwise a negative error number.
482 static int idt_scan_ports(struct idt_ntb_dev *ndev)
484 unsigned char pidx, port, part;
485 u32 data, portsts, partsts;
487 /* Retrieve the local port number */
488 data = idt_nt_read(ndev, IDT_NT_PCIELCAP);
489 ndev->port = GET_FIELD(PCIELCAP_PORTNUM, data);
491 /* Retrieve the local partition number */
492 portsts = idt_sw_read(ndev, portdata_tbl[ndev->port].sts);
493 ndev->part = GET_FIELD(SWPORTxSTS_SWPART, portsts);
495 /* Initialize port/partition -> index tables with invalid values */
496 memset(ndev->port_idx_map, -EINVAL, sizeof(ndev->port_idx_map));
497 memset(ndev->part_idx_map, -EINVAL, sizeof(ndev->part_idx_map));
500 * Walk over all the possible ports checking whether any of them has
501 * NT-function activated
504 for (pidx = 0; pidx < ndev->swcfg->port_cnt; pidx++) {
505 port = ndev->swcfg->ports[pidx];
506 /* Skip local port */
507 if (port == ndev->port)
510 /* Read the port status register to get it partition */
511 portsts = idt_sw_read(ndev, portdata_tbl[port].sts);
512 part = GET_FIELD(SWPORTxSTS_SWPART, portsts);
514 /* Retrieve the partition status */
515 partsts = idt_sw_read(ndev, partdata_tbl[part].sts);
516 /* Check if partition state is active and port has NTB */
517 if (IS_FLD_SET(SWPARTxSTS_STATE, partsts, ACT) &&
518 (IS_FLD_SET(SWPORTxSTS_MODE, portsts, NT) ||
519 IS_FLD_SET(SWPORTxSTS_MODE, portsts, USNT) ||
520 IS_FLD_SET(SWPORTxSTS_MODE, portsts, USNTDMA) ||
521 IS_FLD_SET(SWPORTxSTS_MODE, portsts, NTDMA))) {
522 /* Save the port and partition numbers */
523 ndev->peers[ndev->peer_cnt].port = port;
524 ndev->peers[ndev->peer_cnt].part = part;
525 /* Fill in the port/partition -> index tables */
526 ndev->port_idx_map[port] = ndev->peer_cnt;
527 ndev->part_idx_map[part] = ndev->peer_cnt;
532 dev_dbg(&ndev->ntb.pdev->dev, "Local port: %hhu, num of peers: %hhu\n",
533 ndev->port, ndev->peer_cnt);
535 /* It's useless to have this driver loaded if there is no any peer */
536 if (ndev->peer_cnt == 0) {
537 dev_warn(&ndev->ntb.pdev->dev, "No active peer found\n");
545 * idt_ntb_port_number() - get the local port number
546 * @ntb: NTB device context.
548 * Return: the local port number
550 static int idt_ntb_port_number(struct ntb_dev *ntb)
552 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
558 * idt_ntb_peer_port_count() - get the number of peer ports
559 * @ntb: NTB device context.
561 * Return the count of detected peer NT-functions.
563 * Return: number of peer ports
565 static int idt_ntb_peer_port_count(struct ntb_dev *ntb)
567 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
569 return ndev->peer_cnt;
573 * idt_ntb_peer_port_number() - get peer port by given index
574 * @ntb: NTB device context.
575 * @pidx: Peer port index.
577 * Return: peer port or negative error
579 static int idt_ntb_peer_port_number(struct ntb_dev *ntb, int pidx)
581 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
583 if (pidx < 0 || ndev->peer_cnt <= pidx)
586 /* Return the detected NT-function port number */
587 return ndev->peers[pidx].port;
591 * idt_ntb_peer_port_idx() - get peer port index by given port number
592 * @ntb: NTB device context.
593 * @port: Peer port number.
595 * Internal port -> index table is pre-initialized with -EINVAL values,
596 * so we just need to return it value
598 * Return: peer NT-function port index or negative error
600 static int idt_ntb_peer_port_idx(struct ntb_dev *ntb, int port)
602 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
604 if (port < 0 || IDT_MAX_NR_PORTS <= port)
607 return ndev->port_idx_map[port];
610 /*===========================================================================
611 * 3. Link status operations
612 * There is no any ready-to-use method to have peer ports notified if NTB
613 * link is set up or got down. Instead global signal can be used instead.
614 * In case if any one of ports changes local NTB link state, it sends
615 * global signal and clears corresponding global state bit. Then all the ports
616 * receive a notification of that, so to make client driver being aware of
617 * possible NTB link change.
618 * Additionally each of active NT-functions is subscribed to PCIe-link
619 * state changes of peer ports.
620 *===========================================================================
623 static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev);
626 * idt_init_link() - Initialize NTB link state notification subsystem
627 * @ndev: IDT NTB hardware driver descriptor
629 * Function performs the basic initialization of some global registers
630 * needed to enable IRQ-based notifications of PCIe Link Up/Down and
631 * Global Signal events.
632 * NOTE Since it's not possible to determine when all the NTB peer drivers are
633 * unloaded as well as have those registers accessed concurrently, we must
634 * preinitialize them with the same value and leave it uncleared on local
637 static void idt_init_link(struct idt_ntb_dev *ndev)
639 u32 part_mask, port_mask, se_mask;
642 /* Initialize spin locker of Mapping Table access registers */
643 spin_lock_init(&ndev->mtbl_lock);
645 /* Walk over all detected peers collecting port and partition masks */
646 port_mask = ~BIT(ndev->port);
647 part_mask = ~BIT(ndev->part);
648 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
649 port_mask &= ~BIT(ndev->peers[pidx].port);
650 part_mask &= ~BIT(ndev->peers[pidx].part);
653 /* Clean the Link Up/Down and GLobal Signal status registers */
654 idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1);
655 idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1);
656 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1);
658 /* Unmask NT-activated partitions to receive Global Switch events */
659 idt_sw_write(ndev, IDT_SW_SEPMSK, part_mask);
661 /* Enable PCIe Link Up events of NT-activated ports */
662 idt_sw_write(ndev, IDT_SW_SELINKUPMSK, port_mask);
664 /* Enable PCIe Link Down events of NT-activated ports */
665 idt_sw_write(ndev, IDT_SW_SELINKDNMSK, port_mask);
667 /* Unmask NT-activated partitions to receive Global Signal events */
668 idt_sw_write(ndev, IDT_SW_SEGSIGMSK, part_mask);
670 /* Unmask Link Up/Down and Global Switch Events */
671 se_mask = ~(IDT_SEMSK_LINKUP | IDT_SEMSK_LINKDN | IDT_SEMSK_GSIGNAL);
672 idt_sw_write(ndev, IDT_SW_SEMSK, se_mask);
674 dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events initialized");
678 * idt_deinit_link() - deinitialize link subsystem
679 * @ndev: IDT NTB hardware driver descriptor
681 * Just disable the link back.
683 static void idt_deinit_link(struct idt_ntb_dev *ndev)
685 /* Disable the link */
686 idt_ntb_local_link_disable(ndev);
688 dev_dbg(&ndev->ntb.pdev->dev, "NTB link status events deinitialized");
692 * idt_se_isr() - switch events ISR
693 * @ndev: IDT NTB hardware driver descriptor
694 * @ntint_sts: NT-function interrupt status
696 * This driver doesn't support IDT PCIe-switch dynamic reconfigurations,
697 * Failover capability, etc, so switch events are utilized to notify of
698 * PCIe and NTB link events.
699 * The method is called from PCIe ISR bottom-half routine.
701 static void idt_se_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)
705 /* Read Switch Events status */
706 sests = idt_sw_read(ndev, IDT_SW_SESTS);
708 /* Clean the Link Up/Down and Global Signal status registers */
709 idt_sw_write(ndev, IDT_SW_SELINKUPSTS, (u32)-1);
710 idt_sw_write(ndev, IDT_SW_SELINKDNSTS, (u32)-1);
711 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)-1);
713 /* Clean the corresponding interrupt bit */
714 idt_nt_write(ndev, IDT_NT_NTINTSTS, IDT_NTINTSTS_SEVENT);
716 dev_dbg(&ndev->ntb.pdev->dev, "SE IRQ detected %#08x (SESTS %#08x)",
719 /* Notify the client driver of possible link state change */
720 ntb_link_event(&ndev->ntb);
724 * idt_ntb_local_link_enable() - enable the local NTB link.
725 * @ndev: IDT NTB hardware driver descriptor
727 * In order to enable the NTB link we need:
728 * - enable Completion TLPs translation
729 * - initialize mapping table to enable the Request ID translation
730 * - notify peers of NTB link state change
732 static void idt_ntb_local_link_enable(struct idt_ntb_dev *ndev)
734 u32 reqid, mtbldata = 0;
735 unsigned long irqflags;
737 /* Enable the ID protection and Completion TLPs translation */
738 idt_nt_write(ndev, IDT_NT_NTCTL, IDT_NTCTL_CPEN);
740 /* Retrieve the current Requester ID (Bus:Device:Function) */
741 reqid = idt_nt_read(ndev, IDT_NT_REQIDCAP);
744 * Set the corresponding NT Mapping table entry of port partition index
745 * with the data to perform the Request ID translation
747 mtbldata = SET_FIELD(NTMTBLDATA_REQID, 0, reqid) |
748 SET_FIELD(NTMTBLDATA_PART, 0, ndev->part) |
749 IDT_NTMTBLDATA_VALID;
750 spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
751 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);
752 idt_nt_write(ndev, IDT_NT_NTMTBLDATA, mtbldata);
754 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
756 /* Notify the peers by setting and clearing the global signal bit */
757 idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET);
758 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part);
762 * idt_ntb_local_link_disable() - disable the local NTB link.
763 * @ndev: IDT NTB hardware driver descriptor
765 * In order to enable the NTB link we need:
766 * - disable Completion TLPs translation
767 * - clear corresponding mapping table entry
768 * - notify peers of NTB link state change
770 static void idt_ntb_local_link_disable(struct idt_ntb_dev *ndev)
772 unsigned long irqflags;
774 /* Disable Completion TLPs translation */
775 idt_nt_write(ndev, IDT_NT_NTCTL, 0);
777 /* Clear the corresponding NT Mapping table entry */
778 spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
779 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);
780 idt_nt_write(ndev, IDT_NT_NTMTBLDATA, 0);
782 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
784 /* Notify the peers by setting and clearing the global signal bit */
785 idt_nt_write(ndev, IDT_NT_NTGSIGNAL, IDT_NTGSIGNAL_SET);
786 idt_sw_write(ndev, IDT_SW_SEGSIGSTS, (u32)1 << ndev->part);
790 * idt_ntb_local_link_is_up() - test wethter local NTB link is up
791 * @ndev: IDT NTB hardware driver descriptor
793 * Local link is up under the following conditions:
794 * - Bus mastering is enabled
795 * - NTCTL has Completion TLPs translation enabled
796 * - Mapping table permits Request TLPs translation
797 * NOTE: We don't need to check PCIe link state since it's obviously
798 * up while we are able to communicate with IDT PCIe-switch
800 * Return: true if link is up, otherwise false
802 static bool idt_ntb_local_link_is_up(struct idt_ntb_dev *ndev)
804 unsigned long irqflags;
807 /* Read the local Bus Master Enable status */
808 data = idt_nt_read(ndev, IDT_NT_PCICMDSTS);
809 if (!(data & IDT_PCICMDSTS_BME))
812 /* Read the local Completion TLPs translation enable status */
813 data = idt_nt_read(ndev, IDT_NT_NTCTL);
814 if (!(data & IDT_NTCTL_CPEN))
817 /* Read Mapping table entry corresponding to the local partition */
818 spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
819 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->part);
820 data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);
821 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
823 return !!(data & IDT_NTMTBLDATA_VALID);
827 * idt_ntb_peer_link_is_up() - test whether peer NTB link is up
828 * @ndev: IDT NTB hardware driver descriptor
829 * @pidx: Peer port index
831 * Peer link is up under the following conditions:
833 * - Bus mastering is enabled
834 * - NTCTL has Completion TLPs translation enabled
835 * - Mapping table permits Request TLPs translation
837 * Return: true if link is up, otherwise false
839 static bool idt_ntb_peer_link_is_up(struct idt_ntb_dev *ndev, int pidx)
841 unsigned long irqflags;
845 /* Retrieve the device port number */
846 port = ndev->peers[pidx].port;
848 /* Check whether PCIe link is up */
849 data = idt_sw_read(ndev, portdata_tbl[port].sts);
850 if (!(data & IDT_SWPORTxSTS_LINKUP))
853 /* Check whether bus mastering is enabled on the peer port */
854 data = idt_sw_read(ndev, portdata_tbl[port].pcicmdsts);
855 if (!(data & IDT_PCICMDSTS_BME))
858 /* Check if Completion TLPs translation is enabled on the peer port */
859 data = idt_sw_read(ndev, portdata_tbl[port].ntctl);
860 if (!(data & IDT_NTCTL_CPEN))
863 /* Read Mapping table entry corresponding to the peer partition */
864 spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
865 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, ndev->peers[pidx].part);
866 data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);
867 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
869 return !!(data & IDT_NTMTBLDATA_VALID);
873 * idt_ntb_link_is_up() - get the current ntb link state (NTB API callback)
874 * @ntb: NTB device context.
875 * @speed: OUT - The link speed expressed as PCIe generation number.
876 * @width: OUT - The link width expressed as the number of PCIe lanes.
878 * Get the bitfield of NTB link states for all peer ports
880 * Return: bitfield of indexed ports link state: bit is set/cleared if the
881 * link is up/down respectively.
883 static u64 idt_ntb_link_is_up(struct ntb_dev *ntb,
884 enum ntb_speed *speed, enum ntb_width *width)
886 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
891 /* Retrieve the local link speed and width */
892 if (speed != NULL || width != NULL) {
893 data = idt_nt_read(ndev, IDT_NT_PCIELCTLSTS);
895 *speed = GET_FIELD(PCIELCTLSTS_CLS, data);
897 *width = GET_FIELD(PCIELCTLSTS_NLW, data);
900 /* If local NTB link isn't up then all the links are considered down */
901 if (!idt_ntb_local_link_is_up(ndev))
904 /* Collect all the peer ports link states into the bitfield */
906 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
907 if (idt_ntb_peer_link_is_up(ndev, pidx))
908 status |= ((u64)1 << pidx);
915 * idt_ntb_link_enable() - enable local port ntb link (NTB API callback)
916 * @ntb: NTB device context.
917 * @max_speed: The maximum link speed expressed as PCIe generation number.
918 * @max_width: The maximum link width expressed as the number of PCIe lanes.
920 * Enable just local NTB link. PCIe link parameters are ignored.
922 * Return: always zero.
924 static int idt_ntb_link_enable(struct ntb_dev *ntb, enum ntb_speed speed,
925 enum ntb_width width)
927 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
929 /* Just enable the local NTB link */
930 idt_ntb_local_link_enable(ndev);
932 dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link enabled");
938 * idt_ntb_link_disable() - disable local port ntb link (NTB API callback)
939 * @ntb: NTB device context.
941 * Disable just local NTB link.
943 * Return: always zero.
945 static int idt_ntb_link_disable(struct ntb_dev *ntb)
947 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
949 /* Just disable the local NTB link */
950 idt_ntb_local_link_disable(ndev);
952 dev_dbg(&ndev->ntb.pdev->dev, "Local NTB link disabled");
957 /*=============================================================================
958 * 4. Memory Window operations
960 * IDT PCIe-switches have two types of memory windows: MWs with direct
961 * address translation and MWs with LUT based translation. The first type of
962 * MWs is simple map of corresponding BAR address space to a memory space
963 * of specified target port. So it implemets just ont-to-one mapping. Lookup
964 * table in its turn can map one BAR address space to up to 24 different
965 * memory spaces of different ports.
966 * NT-functions BARs can be turned on to implement either direct or lookup
967 * table based address translations, so:
968 * BAR0 - NT configuration registers space/direct address translation
969 * BAR1 - direct address translation/upper address of BAR0x64
970 * BAR2 - direct address translation/Lookup table with either 12 or 24 entries
971 * BAR3 - direct address translation/upper address of BAR2x64
972 * BAR4 - direct address translation/Lookup table with either 12 or 24 entries
973 * BAR5 - direct address translation/upper address of BAR4x64
974 * Additionally BAR2 and BAR4 can't have 24-entries LUT enabled at the same
975 * time. Since the BARs setup can be rather complicated this driver implements
976 * a scanning algorithm to have all the possible memory windows configuration
979 * NOTE 1 BAR setup must be done before Linux kernel enumerated NT-function
980 * of any port, so this driver would have memory windows configurations fixed.
981 * In this way all initializations must be performed either by platform BIOS
982 * or using EEPROM connected to IDT PCIe-switch master SMBus.
984 * NOTE 2 This driver expects BAR0 mapping NT-function configuration space.
985 * Easy calculation can give us an upper boundary of 29 possible memory windows
986 * per each NT-function if all the BARs are of 32bit type.
987 *=============================================================================
991 * idt_get_mw_count() - get memory window count
992 * @mw_type: Memory window type
994 * Return: number of memory windows with respect to the BAR type
996 static inline unsigned char idt_get_mw_count(enum idt_mw_type mw_type)
1013 * idt_get_mw_name() - get memory window name
1014 * @mw_type: Memory window type
1016 * Return: pointer to a string with name
1018 static inline char *idt_get_mw_name(enum idt_mw_type mw_type)
1035 * idt_scan_mws() - scan memory windows of the port
1036 * @ndev: IDT NTB hardware driver descriptor
1037 * @port: Port to get number of memory windows for
1038 * @mw_cnt: Out - number of memory windows
1040 * It walks over BAR setup registers of the specified port and determines
1041 * the memory windows parameters if any activated.
1043 * Return: array of memory windows
1045 static struct idt_mw_cfg *idt_scan_mws(struct idt_ntb_dev *ndev, int port,
1046 unsigned char *mw_cnt)
1048 struct idt_mw_cfg mws[IDT_MAX_NR_MWS], *ret_mws;
1049 const struct idt_ntb_bar *bars;
1050 enum idt_mw_type mw_type;
1051 unsigned char widx, bidx, en_cnt;
1052 bool bar_64bit = false;
1056 /* Retrieve the array of the BARs registers */
1057 bars = portdata_tbl[port].bars;
1059 /* Scan all the BARs belonging to the port */
1061 for (bidx = 0; bidx < IDT_BAR_CNT; bidx += 1 + bar_64bit) {
1062 /* Read BARSETUP register value */
1063 data = idt_sw_read(ndev, bars[bidx].setup);
1065 /* Skip disabled BARs */
1066 if (!(data & IDT_BARSETUP_EN)) {
1071 /* Skip next BARSETUP if current one has 64bit addressing */
1072 bar_64bit = IS_FLD_SET(BARSETUP_TYPE, data, 64);
1074 /* Skip configuration space mapping BARs */
1075 if (data & IDT_BARSETUP_MODE_CFG)
1078 /* Retrieve MW type/entries count and aperture size */
1079 mw_type = GET_FIELD(BARSETUP_ATRAN, data);
1080 en_cnt = idt_get_mw_count(mw_type);
1081 aprt_size = (u64)1 << GET_FIELD(BARSETUP_SIZE, data);
1083 /* Save configurations of all available memory windows */
1084 for (widx = 0; widx < en_cnt; widx++, (*mw_cnt)++) {
1086 * IDT can expose a limited number of MWs, so it's bug
1087 * to have more than the driver expects
1089 if (*mw_cnt >= IDT_MAX_NR_MWS)
1090 return ERR_PTR(-EINVAL);
1092 /* Save basic MW info */
1093 mws[*mw_cnt].type = mw_type;
1094 mws[*mw_cnt].bar = bidx;
1095 mws[*mw_cnt].idx = widx;
1096 /* It's always DWORD aligned */
1097 mws[*mw_cnt].addr_align = IDT_TRANS_ALIGN;
1098 /* DIR and LUT approachs differently configure MWs */
1099 if (mw_type == IDT_MW_DIR)
1100 mws[*mw_cnt].size_max = aprt_size;
1101 else if (mw_type == IDT_MW_LUT12)
1102 mws[*mw_cnt].size_max = aprt_size / 16;
1104 mws[*mw_cnt].size_max = aprt_size / 32;
1105 mws[*mw_cnt].size_align = (mw_type == IDT_MW_DIR) ?
1106 IDT_DIR_SIZE_ALIGN : mws[*mw_cnt].size_max;
1110 /* Allocate memory for memory window descriptors */
1111 ret_mws = devm_kcalloc(&ndev->ntb.pdev->dev, *mw_cnt, sizeof(*ret_mws),
1114 return ERR_PTR(-ENOMEM);
1116 /* Copy the info of detected memory windows */
1117 memcpy(ret_mws, mws, (*mw_cnt)*sizeof(*ret_mws));
1123 * idt_init_mws() - initialize memory windows subsystem
1124 * @ndev: IDT NTB hardware driver descriptor
1126 * Scan BAR setup registers of local and peer ports to determine the
1127 * outbound and inbound memory windows parameters
1129 * Return: zero on success, otherwise a negative error number
1131 static int idt_init_mws(struct idt_ntb_dev *ndev)
1133 struct idt_ntb_peer *peer;
1136 /* Scan memory windows of the local port */
1137 ndev->mws = idt_scan_mws(ndev, ndev->port, &ndev->mw_cnt);
1138 if (IS_ERR(ndev->mws)) {
1139 dev_err(&ndev->ntb.pdev->dev,
1140 "Failed to scan mws of local port %hhu", ndev->port);
1141 return PTR_ERR(ndev->mws);
1144 /* Scan memory windows of the peer ports */
1145 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
1146 peer = &ndev->peers[pidx];
1147 peer->mws = idt_scan_mws(ndev, peer->port, &peer->mw_cnt);
1148 if (IS_ERR(peer->mws)) {
1149 dev_err(&ndev->ntb.pdev->dev,
1150 "Failed to scan mws of port %hhu", peer->port);
1151 return PTR_ERR(peer->mws);
1155 /* Initialize spin locker of the LUT registers */
1156 spin_lock_init(&ndev->lut_lock);
1158 dev_dbg(&ndev->ntb.pdev->dev, "Outbound and inbound MWs initialized");
1164 * idt_ntb_mw_count() - number of inbound memory windows (NTB API callback)
1165 * @ntb: NTB device context.
1166 * @pidx: Port index of peer device.
1168 * The value is returned for the specified peer, so generally speaking it can
1169 * be different for different port depending on the IDT PCIe-switch
1172 * Return: the number of memory windows.
1174 static int idt_ntb_mw_count(struct ntb_dev *ntb, int pidx)
1176 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1178 if (pidx < 0 || ndev->peer_cnt <= pidx)
1181 return ndev->peers[pidx].mw_cnt;
1185 * idt_ntb_mw_get_align() - inbound memory window parameters (NTB API callback)
1186 * @ntb: NTB device context.
1187 * @pidx: Port index of peer device.
1188 * @widx: Memory window index.
1189 * @addr_align: OUT - the base alignment for translating the memory window
1190 * @size_align: OUT - the size alignment for translating the memory window
1191 * @size_max: OUT - the maximum size of the memory window
1193 * The peer memory window parameters have already been determined, so just
1194 * return the corresponding values, which mustn't change within session.
1196 * Return: Zero on success, otherwise a negative error number.
1198 static int idt_ntb_mw_get_align(struct ntb_dev *ntb, int pidx, int widx,
1199 resource_size_t *addr_align,
1200 resource_size_t *size_align,
1201 resource_size_t *size_max)
1203 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1204 struct idt_ntb_peer *peer;
1206 if (pidx < 0 || ndev->peer_cnt <= pidx)
1209 peer = &ndev->peers[pidx];
1211 if (widx < 0 || peer->mw_cnt <= widx)
1214 if (addr_align != NULL)
1215 *addr_align = peer->mws[widx].addr_align;
1217 if (size_align != NULL)
1218 *size_align = peer->mws[widx].size_align;
1220 if (size_max != NULL)
1221 *size_max = peer->mws[widx].size_max;
1227 * idt_ntb_peer_mw_count() - number of outbound memory windows
1228 * (NTB API callback)
1229 * @ntb: NTB device context.
1231 * Outbound memory windows parameters have been determined based on the
1232 * BAR setup registers value, which are mostly constants within one session.
1234 * Return: the number of memory windows.
1236 static int idt_ntb_peer_mw_count(struct ntb_dev *ntb)
1238 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1240 return ndev->mw_cnt;
1244 * idt_ntb_peer_mw_get_addr() - get map address of an outbound memory window
1245 * (NTB API callback)
1246 * @ntb: NTB device context.
1247 * @widx: Memory window index (within ntb_peer_mw_count() return value).
1248 * @base: OUT - the base address of mapping region.
1249 * @size: OUT - the size of mapping region.
1251 * Return just parameters of BAR resources mapping. Size reflects just the size
1254 * Return: Zero on success, otherwise a negative error number.
1256 static int idt_ntb_peer_mw_get_addr(struct ntb_dev *ntb, int widx,
1257 phys_addr_t *base, resource_size_t *size)
1259 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1261 if (widx < 0 || ndev->mw_cnt <= widx)
1264 /* Mapping address is just properly shifted BAR resource start */
1266 *base = pci_resource_start(ntb->pdev, ndev->mws[widx].bar) +
1267 ndev->mws[widx].idx * ndev->mws[widx].size_max;
1269 /* Mapping size has already been calculated at MWs scanning */
1271 *size = ndev->mws[widx].size_max;
1277 * idt_ntb_peer_mw_set_trans() - set a translation address of a memory window
1278 * (NTB API callback)
1279 * @ntb: NTB device context.
1280 * @pidx: Port index of peer device the translation address received from.
1281 * @widx: Memory window index.
1282 * @addr: The dma address of the shared memory to access.
1283 * @size: The size of the shared memory to access.
1285 * The Direct address translation and LUT base translation is initialized a
1286 * bit differenet. Although the parameters restriction are now determined by
1289 * Return: Zero on success, otherwise an error number.
1291 static int idt_ntb_peer_mw_set_trans(struct ntb_dev *ntb, int pidx, int widx,
1292 u64 addr, resource_size_t size)
1294 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1295 struct idt_mw_cfg *mw_cfg;
1296 u32 data = 0, lutoff = 0;
1298 if (pidx < 0 || ndev->peer_cnt <= pidx)
1301 if (widx < 0 || ndev->mw_cnt <= widx)
1305 * Retrieve the memory window config to make sure the passed arguments
1306 * fit it restrictions
1308 mw_cfg = &ndev->mws[widx];
1309 if (!IS_ALIGNED(addr, mw_cfg->addr_align))
1311 if (!IS_ALIGNED(size, mw_cfg->size_align) || size > mw_cfg->size_max)
1314 /* DIR and LUT based translations are initialized differently */
1315 if (mw_cfg->type == IDT_MW_DIR) {
1316 const struct idt_ntb_bar *bar = &ntdata_tbl.bars[mw_cfg->bar];
1318 /* Set destination partition of translation */
1319 data = idt_nt_read(ndev, bar->setup);
1320 data = SET_FIELD(BARSETUP_TPART, data, ndev->peers[pidx].part);
1321 idt_nt_write(ndev, bar->setup, data);
1322 /* Set translation base address */
1323 idt_nt_write(ndev, bar->ltbase, (u32)addr);
1324 idt_nt_write(ndev, bar->utbase, (u32)(addr >> 32));
1325 /* Set the custom BAR aperture limit */
1326 limit = pci_bus_address(ntb->pdev, mw_cfg->bar) + size;
1327 idt_nt_write(ndev, bar->limit, (u32)limit);
1328 if (IS_FLD_SET(BARSETUP_TYPE, data, 64))
1329 idt_nt_write(ndev, (bar + 1)->limit, (limit >> 32));
1331 unsigned long irqflags;
1332 /* Initialize corresponding LUT entry */
1333 lutoff = SET_FIELD(LUTOFFSET_INDEX, 0, mw_cfg->idx) |
1334 SET_FIELD(LUTOFFSET_BAR, 0, mw_cfg->bar);
1335 data = SET_FIELD(LUTUDATA_PART, 0, ndev->peers[pidx].part) |
1337 spin_lock_irqsave(&ndev->lut_lock, irqflags);
1338 idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff);
1339 idt_nt_write(ndev, IDT_NT_LUTLDATA, (u32)addr);
1340 idt_nt_write(ndev, IDT_NT_LUTMDATA, (u32)(addr >> 32));
1341 idt_nt_write(ndev, IDT_NT_LUTUDATA, data);
1343 spin_unlock_irqrestore(&ndev->lut_lock, irqflags);
1344 /* Limit address isn't specified since size is fixed for LUT */
1351 * idt_ntb_peer_mw_clear_trans() - clear the outbound MW translation address
1352 * (NTB API callback)
1353 * @ntb: NTB device context.
1354 * @pidx: Port index of peer device.
1355 * @widx: Memory window index.
1357 * It effectively disables the translation over the specified outbound MW.
1359 * Return: Zero on success, otherwise an error number.
1361 static int idt_ntb_peer_mw_clear_trans(struct ntb_dev *ntb, int pidx,
1364 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1365 struct idt_mw_cfg *mw_cfg;
1367 if (pidx < 0 || ndev->peer_cnt <= pidx)
1370 if (widx < 0 || ndev->mw_cnt <= widx)
1373 mw_cfg = &ndev->mws[widx];
1375 /* DIR and LUT based translations are initialized differently */
1376 if (mw_cfg->type == IDT_MW_DIR) {
1377 const struct idt_ntb_bar *bar = &ntdata_tbl.bars[mw_cfg->bar];
1379 /* Read BARSETUP to check BAR type */
1380 data = idt_nt_read(ndev, bar->setup);
1381 /* Disable translation by specifying zero BAR limit */
1382 idt_nt_write(ndev, bar->limit, 0);
1383 if (IS_FLD_SET(BARSETUP_TYPE, data, 64))
1384 idt_nt_write(ndev, (bar + 1)->limit, 0);
1386 unsigned long irqflags;
1388 /* Clear the corresponding LUT entry up */
1389 lutoff = SET_FIELD(LUTOFFSET_INDEX, 0, mw_cfg->idx) |
1390 SET_FIELD(LUTOFFSET_BAR, 0, mw_cfg->bar);
1391 spin_lock_irqsave(&ndev->lut_lock, irqflags);
1392 idt_nt_write(ndev, IDT_NT_LUTOFFSET, lutoff);
1393 idt_nt_write(ndev, IDT_NT_LUTLDATA, 0);
1394 idt_nt_write(ndev, IDT_NT_LUTMDATA, 0);
1395 idt_nt_write(ndev, IDT_NT_LUTUDATA, 0);
1397 spin_unlock_irqrestore(&ndev->lut_lock, irqflags);
1403 /*=============================================================================
1404 * 5. Doorbell operations
1406 * Doorbell functionality of IDT PCIe-switches is pretty unusual. First of
1407 * all there is global doorbell register which state can be changed by any
1408 * NT-function of the IDT device in accordance with global permissions. These
1409 * permissions configs are not supported by NTB API, so it must be done by
1410 * either BIOS or EEPROM settings. In the same way the state of the global
1411 * doorbell is reflected to the NT-functions local inbound doorbell registers.
1412 * It can lead to situations when client driver sets some peer doorbell bits
1413 * and get them bounced back to local inbound doorbell if permissions are
1415 * Secondly there is just one IRQ vector for Doorbell, Message, Temperature
1416 * and Switch events, so if client driver left any of Doorbell bits set and
1417 * some other event occurred, the driver will be notified of Doorbell event
1419 *=============================================================================
1423 * idt_db_isr() - doorbell event ISR
1424 * @ndev: IDT NTB hardware driver descriptor
1425 * @ntint_sts: NT-function interrupt status
1427 * Doorbell event happans when DBELL bit of NTINTSTS switches from 0 to 1.
1428 * It happens only when unmasked doorbell bits are set to ones on completely
1429 * zeroed doorbell register.
1430 * The method is called from PCIe ISR bottom-half routine.
1432 static void idt_db_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)
1435 * Doorbell IRQ status will be cleaned only when client
1436 * driver unsets all the doorbell bits.
1438 dev_dbg(&ndev->ntb.pdev->dev, "DB IRQ detected %#08x", ntint_sts);
1440 /* Notify the client driver of possible doorbell state change */
1441 ntb_db_event(&ndev->ntb, 0);
1445 * idt_ntb_db_valid_mask() - get a mask of doorbell bits supported by the ntb
1446 * (NTB API callback)
1447 * @ntb: NTB device context.
1449 * IDT PCIe-switches expose just one Doorbell register of DWORD size.
1451 * Return: A mask of doorbell bits supported by the ntb.
1453 static u64 idt_ntb_db_valid_mask(struct ntb_dev *ntb)
1455 return IDT_DBELL_MASK;
1459 * idt_ntb_db_read() - read the local doorbell register (NTB API callback)
1460 * @ntb: NTB device context.
1462 * There is just on inbound doorbell register of each NT-function, so
1463 * this method return it value.
1465 * Return: The bits currently set in the local doorbell register.
1467 static u64 idt_ntb_db_read(struct ntb_dev *ntb)
1469 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1471 return idt_nt_read(ndev, IDT_NT_INDBELLSTS);
1475 * idt_ntb_db_clear() - clear bits in the local doorbell register
1476 * (NTB API callback)
1477 * @ntb: NTB device context.
1478 * @db_bits: Doorbell bits to clear.
1480 * Clear bits of inbound doorbell register by writing ones to it.
1482 * NOTE! Invalid bits are always considered cleared so it's not an error
1483 * to clear them over.
1485 * Return: always zero as success.
1487 static int idt_ntb_db_clear(struct ntb_dev *ntb, u64 db_bits)
1489 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1491 idt_nt_write(ndev, IDT_NT_INDBELLSTS, (u32)db_bits);
1497 * idt_ntb_db_read_mask() - read the local doorbell mask (NTB API callback)
1498 * @ntb: NTB device context.
1500 * Each inbound doorbell bit can be masked from generating IRQ by setting
1501 * the corresponding bit in inbound doorbell mask. So this method returns
1502 * the value of the register.
1504 * Return: The bits currently set in the local doorbell mask register.
1506 static u64 idt_ntb_db_read_mask(struct ntb_dev *ntb)
1508 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1510 return idt_nt_read(ndev, IDT_NT_INDBELLMSK);
1514 * idt_ntb_db_set_mask() - set bits in the local doorbell mask
1515 * (NTB API callback)
1516 * @ntb: NTB device context.
1517 * @db_bits: Doorbell mask bits to set.
1519 * The inbound doorbell register mask value must be read, then OR'ed with
1520 * passed field and only then set back.
1522 * Return: zero on success, negative error if invalid argument passed.
1524 static int idt_ntb_db_set_mask(struct ntb_dev *ntb, u64 db_bits)
1526 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1528 return idt_reg_set_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock,
1529 IDT_DBELL_MASK, db_bits);
1533 * idt_ntb_db_clear_mask() - clear bits in the local doorbell mask
1534 * (NTB API callback)
1535 * @ntb: NTB device context.
1536 * @db_bits: Doorbell bits to clear.
1538 * The method just clears the set bits up in accordance with the passed
1539 * bitfield. IDT PCIe-switch shall generate an interrupt if there hasn't
1540 * been any unmasked bit set before current unmasking. Otherwise IRQ won't
1541 * be generated since there is only one IRQ vector for all doorbells.
1543 * Return: always zero as success
1545 static int idt_ntb_db_clear_mask(struct ntb_dev *ntb, u64 db_bits)
1547 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1549 idt_reg_clear_bits(ndev, IDT_NT_INDBELLMSK, &ndev->db_mask_lock,
1556 * idt_ntb_peer_db_set() - set bits in the peer doorbell register
1557 * (NTB API callback)
1558 * @ntb: NTB device context.
1559 * @db_bits: Doorbell bits to set.
1561 * IDT PCIe-switches exposes local outbound doorbell register to change peer
1562 * inbound doorbell register state.
1564 * Return: zero on success, negative error if invalid argument passed.
1566 static int idt_ntb_peer_db_set(struct ntb_dev *ntb, u64 db_bits)
1568 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1570 if (db_bits & ~(u64)IDT_DBELL_MASK)
1573 idt_nt_write(ndev, IDT_NT_OUTDBELLSET, (u32)db_bits);
1577 /*=============================================================================
1578 * 6. Messaging operations
1580 * Each NT-function of IDT PCIe-switch has four inbound and four outbound
1581 * message registers. Each outbound message register can be connected to one or
1582 * even more than one peer inbound message registers by setting global
1583 * configurations. Since NTB API permits one-on-one message registers mapping
1584 * only, the driver acts in according with that restriction.
1585 *=============================================================================
1589 * idt_init_msg() - initialize messaging interface
1590 * @ndev: IDT NTB hardware driver descriptor
1592 * Just initialize the message registers routing tables locker.
1594 static void idt_init_msg(struct idt_ntb_dev *ndev)
1598 /* Init the messages routing table lockers */
1599 for (midx = 0; midx < IDT_MSG_CNT; midx++)
1600 spin_lock_init(&ndev->msg_locks[midx]);
1602 dev_dbg(&ndev->ntb.pdev->dev, "NTB Messaging initialized");
1606 * idt_msg_isr() - message event ISR
1607 * @ndev: IDT NTB hardware driver descriptor
1608 * @ntint_sts: NT-function interrupt status
1610 * Message event happens when MSG bit of NTINTSTS switches from 0 to 1.
1611 * It happens only when unmasked message status bits are set to ones on
1612 * completely zeroed message status register.
1613 * The method is called from PCIe ISR bottom-half routine.
1615 static void idt_msg_isr(struct idt_ntb_dev *ndev, u32 ntint_sts)
1618 * Message IRQ status will be cleaned only when client
1619 * driver unsets all the message status bits.
1621 dev_dbg(&ndev->ntb.pdev->dev, "Message IRQ detected %#08x", ntint_sts);
1623 /* Notify the client driver of possible message status change */
1624 ntb_msg_event(&ndev->ntb);
1628 * idt_ntb_msg_count() - get the number of message registers (NTB API callback)
1629 * @ntb: NTB device context.
1631 * IDT PCIe-switches support four message registers.
1633 * Return: the number of message registers.
1635 static int idt_ntb_msg_count(struct ntb_dev *ntb)
1641 * idt_ntb_msg_inbits() - get a bitfield of inbound message registers status
1642 * (NTB API callback)
1643 * @ntb: NTB device context.
1645 * NT message status register is shared between inbound and outbound message
1648 * Return: bitfield of inbound message registers.
1650 static u64 idt_ntb_msg_inbits(struct ntb_dev *ntb)
1652 return (u64)IDT_INMSG_MASK;
1656 * idt_ntb_msg_outbits() - get a bitfield of outbound message registers status
1657 * (NTB API callback)
1658 * @ntb: NTB device context.
1660 * NT message status register is shared between inbound and outbound message
1663 * Return: bitfield of outbound message registers.
1665 static u64 idt_ntb_msg_outbits(struct ntb_dev *ntb)
1667 return (u64)IDT_OUTMSG_MASK;
1671 * idt_ntb_msg_read_sts() - read the message registers status (NTB API callback)
1672 * @ntb: NTB device context.
1674 * IDT PCIe-switches expose message status registers to notify drivers of
1675 * incoming data and failures in case if peer message register isn't freed.
1677 * Return: status bits of message registers
1679 static u64 idt_ntb_msg_read_sts(struct ntb_dev *ntb)
1681 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1683 return idt_nt_read(ndev, IDT_NT_MSGSTS);
1687 * idt_ntb_msg_clear_sts() - clear status bits of message registers
1688 * (NTB API callback)
1689 * @ntb: NTB device context.
1690 * @sts_bits: Status bits to clear.
1692 * Clear bits in the status register by writing ones.
1694 * NOTE! Invalid bits are always considered cleared so it's not an error
1695 * to clear them over.
1697 * Return: always zero as success.
1699 static int idt_ntb_msg_clear_sts(struct ntb_dev *ntb, u64 sts_bits)
1701 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1703 idt_nt_write(ndev, IDT_NT_MSGSTS, sts_bits);
1709 * idt_ntb_msg_set_mask() - set mask of message register status bits
1710 * (NTB API callback)
1711 * @ntb: NTB device context.
1712 * @mask_bits: Mask bits.
1714 * Mask the message status bits from raising an IRQ.
1716 * Return: zero on success, negative error if invalid argument passed.
1718 static int idt_ntb_msg_set_mask(struct ntb_dev *ntb, u64 mask_bits)
1720 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1722 return idt_reg_set_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock,
1723 IDT_MSG_MASK, mask_bits);
1727 * idt_ntb_msg_clear_mask() - clear message registers mask
1728 * (NTB API callback)
1729 * @ntb: NTB device context.
1730 * @mask_bits: Mask bits.
1732 * Clear mask of message status bits IRQs.
1734 * Return: always zero as success.
1736 static int idt_ntb_msg_clear_mask(struct ntb_dev *ntb, u64 mask_bits)
1738 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1740 idt_reg_clear_bits(ndev, IDT_NT_MSGSTSMSK, &ndev->msg_mask_lock,
1747 * idt_ntb_msg_read() - read message register with specified index
1748 * (NTB API callback)
1749 * @ntb: NTB device context.
1750 * @pidx: OUT - Port index of peer device a message retrieved from
1751 * @midx: Message register index
1753 * Read data from the specified message register and source register.
1755 * Return: inbound message register value.
1757 static u32 idt_ntb_msg_read(struct ntb_dev *ntb, int *pidx, int midx)
1759 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1761 if (midx < 0 || IDT_MSG_CNT <= midx)
1764 /* Retrieve source port index of the message */
1768 srcpart = idt_nt_read(ndev, ntdata_tbl.msgs[midx].src);
1769 *pidx = ndev->part_idx_map[srcpart];
1771 /* Sanity check partition index (for initial case) */
1772 if (*pidx == -EINVAL)
1776 /* Retrieve data of the corresponding message register */
1777 return idt_nt_read(ndev, ntdata_tbl.msgs[midx].in);
1781 * idt_ntb_peer_msg_write() - write data to the specified message register
1782 * (NTB API callback)
1783 * @ntb: NTB device context.
1784 * @pidx: Port index of peer device a message being sent to
1785 * @midx: Message register index
1786 * @msg: Data to send
1788 * Just try to send data to a peer. Message status register should be
1789 * checked by client driver.
1791 * Return: zero on success, negative error if invalid argument passed.
1793 static int idt_ntb_peer_msg_write(struct ntb_dev *ntb, int pidx, int midx,
1796 struct idt_ntb_dev *ndev = to_ndev_ntb(ntb);
1797 unsigned long irqflags;
1800 if (midx < 0 || IDT_MSG_CNT <= midx)
1803 if (pidx < 0 || ndev->peer_cnt <= pidx)
1806 /* Collect the routing information */
1807 swpmsgctl = SET_FIELD(SWPxMSGCTL_REG, 0, midx) |
1808 SET_FIELD(SWPxMSGCTL_PART, 0, ndev->peers[pidx].part);
1810 /* Lock the messages routing table of the specified register */
1811 spin_lock_irqsave(&ndev->msg_locks[midx], irqflags);
1812 /* Set the route and send the data */
1813 idt_sw_write(ndev, partdata_tbl[ndev->part].msgctl[midx], swpmsgctl);
1814 idt_nt_write(ndev, ntdata_tbl.msgs[midx].out, msg);
1816 /* Unlock the messages routing table */
1817 spin_unlock_irqrestore(&ndev->msg_locks[midx], irqflags);
1819 /* Client driver shall check the status register */
1823 /*=============================================================================
1824 * 7. Temperature sensor operations
1826 * IDT PCIe-switch has an embedded temperature sensor, which can be used to
1827 * check current chip core temperature. Since a workload environment can be
1828 * different on different platforms, an offset and ADC/filter settings can be
1829 * specified. Although the offset configuration is only exposed to the sysfs
1830 * hwmon interface at the moment. The rest of the settings can be adjusted
1831 * for instance by the BIOS/EEPROM firmware.
1832 *=============================================================================
1836 * idt_get_deg() - convert millidegree Celsius value to just degree
1837 * @mdegC: IN - millidegree Celsius value
1839 * Return: Degree corresponding to the passed millidegree value
1841 static inline s8 idt_get_deg(long mdegC)
1843 return mdegC / 1000;
1847 * idt_get_frac() - retrieve 0/0.5 fraction of the millidegree Celsius value
1848 * @mdegC: IN - millidegree Celsius value
1850 * Return: 0/0.5 degree fraction of the passed millidegree value
1852 static inline u8 idt_get_deg_frac(long mdegC)
1854 return (mdegC % 1000) >= 500 ? 5 : 0;
1858 * idt_get_temp_fmt() - convert millidegree Celsius value to 0:7:1 format
1859 * @mdegC: IN - millidegree Celsius value
1861 * Return: 0:7:1 format acceptable by the IDT temperature sensor
1863 static inline u8 idt_temp_get_fmt(long mdegC)
1865 return (idt_get_deg(mdegC) << 1) | (idt_get_deg_frac(mdegC) ? 1 : 0);
1869 * idt_get_temp_sval() - convert temp sample to signed millidegree Celsius
1870 * @data: IN - shifted to LSB 8-bits temperature sample
1872 * Return: signed millidegree Celsius
1874 static inline long idt_get_temp_sval(u32 data)
1876 return ((s8)data / 2) * 1000 + (data & 0x1 ? 500 : 0);
1880 * idt_get_temp_sval() - convert temp sample to unsigned millidegree Celsius
1881 * @data: IN - shifted to LSB 8-bits temperature sample
1883 * Return: unsigned millidegree Celsius
1885 static inline long idt_get_temp_uval(u32 data)
1887 return (data / 2) * 1000 + (data & 0x1 ? 500 : 0);
1891 * idt_read_temp() - read temperature from chip sensor
1892 * @ntb: NTB device context.
1893 * @type: IN - type of the temperature value to read
1894 * @val: OUT - integer value of temperature in millidegree Celsius
1896 static void idt_read_temp(struct idt_ntb_dev *ndev,
1897 const enum idt_temp_val type, long *val)
1901 /* Alter the temperature field in accordance with the passed type */
1904 data = GET_FIELD(TMPSTS_TEMP,
1905 idt_sw_read(ndev, IDT_SW_TMPSTS));
1908 data = GET_FIELD(TMPSTS_LTEMP,
1909 idt_sw_read(ndev, IDT_SW_TMPSTS));
1912 data = GET_FIELD(TMPSTS_HTEMP,
1913 idt_sw_read(ndev, IDT_SW_TMPSTS));
1915 case IDT_TEMP_OFFSET:
1916 /* This is the only field with signed 0:7:1 format */
1917 data = GET_FIELD(TMPADJ_OFFSET,
1918 idt_sw_read(ndev, IDT_SW_TMPADJ));
1919 *val = idt_get_temp_sval(data);
1922 data = GET_FIELD(TMPSTS_TEMP,
1923 idt_sw_read(ndev, IDT_SW_TMPSTS));
1927 /* The rest of the fields accept unsigned 0:7:1 format */
1928 *val = idt_get_temp_uval(data);
1932 * idt_write_temp() - write temperature to the chip sensor register
1933 * @ntb: NTB device context.
1934 * @type: IN - type of the temperature value to change
1935 * @val: IN - integer value of temperature in millidegree Celsius
1937 static void idt_write_temp(struct idt_ntb_dev *ndev,
1938 const enum idt_temp_val type, const long val)
1944 /* Retrieve the properly formatted temperature value */
1945 fmt = idt_temp_get_fmt(val);
1947 mutex_lock(&ndev->hwmon_mtx);
1950 reg = IDT_SW_TMPALARM;
1951 data = SET_FIELD(TMPALARM_LTEMP, idt_sw_read(ndev, reg), fmt) &
1952 ~IDT_TMPALARM_IRQ_MASK;
1955 reg = IDT_SW_TMPALARM;
1956 data = SET_FIELD(TMPALARM_HTEMP, idt_sw_read(ndev, reg), fmt) &
1957 ~IDT_TMPALARM_IRQ_MASK;
1959 case IDT_TEMP_OFFSET:
1960 reg = IDT_SW_TMPADJ;
1961 data = SET_FIELD(TMPADJ_OFFSET, idt_sw_read(ndev, reg), fmt);
1964 goto inval_spin_unlock;
1967 idt_sw_write(ndev, reg, data);
1970 mutex_unlock(&ndev->hwmon_mtx);
1974 * idt_sysfs_show_temp() - printout corresponding temperature value
1975 * @dev: Pointer to the NTB device structure
1976 * @da: Sensor device attribute structure
1977 * @buf: Buffer to print temperature out
1979 * Return: Number of written symbols or negative error
1981 static ssize_t idt_sysfs_show_temp(struct device *dev,
1982 struct device_attribute *da, char *buf)
1984 struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
1985 struct idt_ntb_dev *ndev = dev_get_drvdata(dev);
1986 enum idt_temp_val type = attr->index;
1989 idt_read_temp(ndev, type, &mdeg);
1990 return sprintf(buf, "%ld\n", mdeg);
1994 * idt_sysfs_set_temp() - set corresponding temperature value
1995 * @dev: Pointer to the NTB device structure
1996 * @da: Sensor device attribute structure
1997 * @buf: Buffer to print temperature out
1998 * @count: Size of the passed buffer
2000 * Return: Number of written symbols or negative error
2002 static ssize_t idt_sysfs_set_temp(struct device *dev,
2003 struct device_attribute *da, const char *buf,
2006 struct sensor_device_attribute *attr = to_sensor_dev_attr(da);
2007 struct idt_ntb_dev *ndev = dev_get_drvdata(dev);
2008 enum idt_temp_val type = attr->index;
2012 ret = kstrtol(buf, 10, &mdeg);
2016 /* Clamp the passed value in accordance with the type */
2017 if (type == IDT_TEMP_OFFSET)
2018 mdeg = clamp_val(mdeg, IDT_TEMP_MIN_OFFSET,
2019 IDT_TEMP_MAX_OFFSET);
2021 mdeg = clamp_val(mdeg, IDT_TEMP_MIN_MDEG, IDT_TEMP_MAX_MDEG);
2023 idt_write_temp(ndev, type, mdeg);
2029 * idt_sysfs_reset_hist() - reset temperature history
2030 * @dev: Pointer to the NTB device structure
2031 * @da: Sensor device attribute structure
2032 * @buf: Buffer to print temperature out
2033 * @count: Size of the passed buffer
2035 * Return: Number of written symbols or negative error
2037 static ssize_t idt_sysfs_reset_hist(struct device *dev,
2038 struct device_attribute *da,
2039 const char *buf, size_t count)
2041 struct idt_ntb_dev *ndev = dev_get_drvdata(dev);
2043 /* Just set the maximal value to the lowest temperature field and
2044 * minimal value to the highest temperature field
2046 idt_write_temp(ndev, IDT_TEMP_LOW, IDT_TEMP_MAX_MDEG);
2047 idt_write_temp(ndev, IDT_TEMP_HIGH, IDT_TEMP_MIN_MDEG);
2053 * Hwmon IDT sysfs attributes
2055 static SENSOR_DEVICE_ATTR(temp1_input, 0444, idt_sysfs_show_temp, NULL,
2057 static SENSOR_DEVICE_ATTR(temp1_lowest, 0444, idt_sysfs_show_temp, NULL,
2059 static SENSOR_DEVICE_ATTR(temp1_highest, 0444, idt_sysfs_show_temp, NULL,
2061 static SENSOR_DEVICE_ATTR(temp1_offset, 0644, idt_sysfs_show_temp,
2062 idt_sysfs_set_temp, IDT_TEMP_OFFSET);
2063 static DEVICE_ATTR(temp1_reset_history, 0200, NULL, idt_sysfs_reset_hist);
2066 * Hwmon IDT sysfs attributes group
2068 static struct attribute *idt_temp_attrs[] = {
2069 &sensor_dev_attr_temp1_input.dev_attr.attr,
2070 &sensor_dev_attr_temp1_lowest.dev_attr.attr,
2071 &sensor_dev_attr_temp1_highest.dev_attr.attr,
2072 &sensor_dev_attr_temp1_offset.dev_attr.attr,
2073 &dev_attr_temp1_reset_history.attr,
2076 ATTRIBUTE_GROUPS(idt_temp);
2079 * idt_init_temp() - initialize temperature sensor interface
2080 * @ndev: IDT NTB hardware driver descriptor
2082 * Simple sensor initializarion method is responsible for device switching
2083 * on and resource management based hwmon interface registration. Note, that
2084 * since the device is shared we won't disable it on remove, but leave it
2085 * working until the system is powered off.
2087 static void idt_init_temp(struct idt_ntb_dev *ndev)
2089 struct device *hwmon;
2091 /* Enable sensor if it hasn't been already */
2092 idt_sw_write(ndev, IDT_SW_TMPCTL, 0x0);
2094 /* Initialize hwmon interface fields */
2095 mutex_init(&ndev->hwmon_mtx);
2097 hwmon = devm_hwmon_device_register_with_groups(&ndev->ntb.pdev->dev,
2098 ndev->swcfg->name, ndev, idt_temp_groups);
2099 if (IS_ERR(hwmon)) {
2100 dev_err(&ndev->ntb.pdev->dev, "Couldn't create hwmon device");
2104 dev_dbg(&ndev->ntb.pdev->dev, "Temperature HWmon interface registered");
2107 /*=============================================================================
2108 * 8. ISRs related operations
2110 * IDT PCIe-switch has strangely developed IRQ system. There is just one
2111 * interrupt vector for doorbell and message registers. So the hardware driver
2112 * can't determine actual source of IRQ if, for example, message event happened
2113 * while any of unmasked doorbell is still set. The similar situation may be if
2114 * switch or temperature sensor events pop up. The difference is that SEVENT
2115 * and TMPSENSOR bits of NT interrupt status register can be cleaned by
2116 * IRQ handler so a next interrupt request won't have false handling of
2117 * corresponding events.
2118 * The hardware driver has only bottom-half handler of the IRQ, since if any
2119 * of events happened the device won't raise it again before the last one is
2120 * handled by clearing of corresponding NTINTSTS bit.
2121 *=============================================================================
2124 static irqreturn_t idt_thread_isr(int irq, void *devid);
2127 * idt_init_isr() - initialize PCIe interrupt handler
2128 * @ndev: IDT NTB hardware driver descriptor
2130 * Return: zero on success, otherwise a negative error number.
2132 static int idt_init_isr(struct idt_ntb_dev *ndev)
2134 struct pci_dev *pdev = ndev->ntb.pdev;
2138 /* Allocate just one interrupt vector for the ISR */
2139 ret = pci_alloc_irq_vectors(pdev, 1, 1, PCI_IRQ_MSI | PCI_IRQ_LEGACY);
2141 dev_err(&pdev->dev, "Failed to allocate IRQ vector");
2145 /* Retrieve the IRQ vector */
2146 ret = pci_irq_vector(pdev, 0);
2148 dev_err(&pdev->dev, "Failed to get IRQ vector");
2149 goto err_free_vectors;
2152 /* Set the IRQ handler */
2153 ret = devm_request_threaded_irq(&pdev->dev, ret, NULL, idt_thread_isr,
2154 IRQF_ONESHOT, NTB_IRQNAME, ndev);
2156 dev_err(&pdev->dev, "Failed to set MSI IRQ handler, %d", ret);
2157 goto err_free_vectors;
2160 /* Unmask Message/Doorbell/SE interrupts */
2161 ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) & ~IDT_NTINTMSK_ALL;
2162 idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask);
2164 /* From now on the interrupts are enabled */
2165 dev_dbg(&pdev->dev, "NTB interrupts initialized");
2170 pci_free_irq_vectors(pdev);
2176 * idt_deinit_ist() - deinitialize PCIe interrupt handler
2177 * @ndev: IDT NTB hardware driver descriptor
2179 * Disable corresponding interrupts and free allocated IRQ vectors.
2181 static void idt_deinit_isr(struct idt_ntb_dev *ndev)
2183 struct pci_dev *pdev = ndev->ntb.pdev;
2186 /* Mask interrupts back */
2187 ntint_mask = idt_nt_read(ndev, IDT_NT_NTINTMSK) | IDT_NTINTMSK_ALL;
2188 idt_nt_write(ndev, IDT_NT_NTINTMSK, ntint_mask);
2190 /* Manually free IRQ otherwise PCI free irq vectors will fail */
2191 devm_free_irq(&pdev->dev, pci_irq_vector(pdev, 0), ndev);
2193 /* Free allocated IRQ vectors */
2194 pci_free_irq_vectors(pdev);
2196 dev_dbg(&pdev->dev, "NTB interrupts deinitialized");
2200 * idt_thread_isr() - NT function interrupts handler
2202 * @devid: Custom buffer
2204 * It reads current NT interrupts state register and handles all the event
2206 * The method is bottom-half routine of actual default PCIe IRQ handler.
2208 static irqreturn_t idt_thread_isr(int irq, void *devid)
2210 struct idt_ntb_dev *ndev = devid;
2211 bool handled = false;
2214 /* Read the NT interrupts status register */
2215 ntint_sts = idt_nt_read(ndev, IDT_NT_NTINTSTS);
2217 /* Handle messaging interrupts */
2218 if (ntint_sts & IDT_NTINTSTS_MSG) {
2219 idt_msg_isr(ndev, ntint_sts);
2223 /* Handle doorbell interrupts */
2224 if (ntint_sts & IDT_NTINTSTS_DBELL) {
2225 idt_db_isr(ndev, ntint_sts);
2229 /* Handle switch event interrupts */
2230 if (ntint_sts & IDT_NTINTSTS_SEVENT) {
2231 idt_se_isr(ndev, ntint_sts);
2235 dev_dbg(&ndev->ntb.pdev->dev, "IDT IRQs 0x%08x handled", ntint_sts);
2237 return handled ? IRQ_HANDLED : IRQ_NONE;
2240 /*===========================================================================
2241 * 9. NTB hardware driver initialization
2242 *===========================================================================
2246 * NTB API operations
2248 static const struct ntb_dev_ops idt_ntb_ops = {
2249 .port_number = idt_ntb_port_number,
2250 .peer_port_count = idt_ntb_peer_port_count,
2251 .peer_port_number = idt_ntb_peer_port_number,
2252 .peer_port_idx = idt_ntb_peer_port_idx,
2253 .link_is_up = idt_ntb_link_is_up,
2254 .link_enable = idt_ntb_link_enable,
2255 .link_disable = idt_ntb_link_disable,
2256 .mw_count = idt_ntb_mw_count,
2257 .mw_get_align = idt_ntb_mw_get_align,
2258 .peer_mw_count = idt_ntb_peer_mw_count,
2259 .peer_mw_get_addr = idt_ntb_peer_mw_get_addr,
2260 .peer_mw_set_trans = idt_ntb_peer_mw_set_trans,
2261 .peer_mw_clear_trans = idt_ntb_peer_mw_clear_trans,
2262 .db_valid_mask = idt_ntb_db_valid_mask,
2263 .db_read = idt_ntb_db_read,
2264 .db_clear = idt_ntb_db_clear,
2265 .db_read_mask = idt_ntb_db_read_mask,
2266 .db_set_mask = idt_ntb_db_set_mask,
2267 .db_clear_mask = idt_ntb_db_clear_mask,
2268 .peer_db_set = idt_ntb_peer_db_set,
2269 .msg_count = idt_ntb_msg_count,
2270 .msg_inbits = idt_ntb_msg_inbits,
2271 .msg_outbits = idt_ntb_msg_outbits,
2272 .msg_read_sts = idt_ntb_msg_read_sts,
2273 .msg_clear_sts = idt_ntb_msg_clear_sts,
2274 .msg_set_mask = idt_ntb_msg_set_mask,
2275 .msg_clear_mask = idt_ntb_msg_clear_mask,
2276 .msg_read = idt_ntb_msg_read,
2277 .peer_msg_write = idt_ntb_peer_msg_write
2281 * idt_register_device() - register IDT NTB device
2282 * @ndev: IDT NTB hardware driver descriptor
2284 * Return: zero on success, otherwise a negative error number.
2286 static int idt_register_device(struct idt_ntb_dev *ndev)
2290 /* Initialize the rest of NTB device structure and register it */
2291 ndev->ntb.ops = &idt_ntb_ops;
2292 ndev->ntb.topo = NTB_TOPO_SWITCH;
2294 ret = ntb_register_device(&ndev->ntb);
2296 dev_err(&ndev->ntb.pdev->dev, "Failed to register NTB device");
2300 dev_dbg(&ndev->ntb.pdev->dev, "NTB device successfully registered");
2306 * idt_unregister_device() - unregister IDT NTB device
2307 * @ndev: IDT NTB hardware driver descriptor
2309 static void idt_unregister_device(struct idt_ntb_dev *ndev)
2311 /* Just unregister the NTB device */
2312 ntb_unregister_device(&ndev->ntb);
2314 dev_dbg(&ndev->ntb.pdev->dev, "NTB device unregistered");
2317 /*=============================================================================
2318 * 10. DebugFS node initialization
2319 *=============================================================================
2322 static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf,
2323 size_t count, loff_t *offp);
2326 * Driver DebugFS info file operations
2328 static const struct file_operations idt_dbgfs_info_ops = {
2329 .owner = THIS_MODULE,
2330 .open = simple_open,
2331 .read = idt_dbgfs_info_read
2335 * idt_dbgfs_info_read() - DebugFS read info node callback
2336 * @file: File node descriptor.
2337 * @ubuf: User-space buffer to put data to
2338 * @count: Size of the buffer
2339 * @offp: Offset within the buffer
2341 static ssize_t idt_dbgfs_info_read(struct file *filp, char __user *ubuf,
2342 size_t count, loff_t *offp)
2344 struct idt_ntb_dev *ndev = filp->private_data;
2345 unsigned char idx, pidx, cnt;
2346 unsigned long irqflags, mdeg;
2347 ssize_t ret = 0, off = 0;
2348 enum ntb_speed speed;
2349 enum ntb_width width;
2354 /* Lets limit the buffer size the way the Intel/AMD drivers do */
2355 size = min_t(size_t, count, 0x1000U);
2357 /* Allocate the memory for the buffer */
2358 strbuf = kmalloc(size, GFP_KERNEL);
2362 /* Put the data into the string buffer */
2363 off += scnprintf(strbuf + off, size - off,
2364 "\n\t\tIDT NTB device Information:\n\n");
2366 /* General local device configurations */
2367 off += scnprintf(strbuf + off, size - off,
2368 "Local Port %hhu, Partition %hhu\n", ndev->port, ndev->part);
2370 /* Peer ports information */
2371 off += scnprintf(strbuf + off, size - off, "Peers:\n");
2372 for (idx = 0; idx < ndev->peer_cnt; idx++) {
2373 off += scnprintf(strbuf + off, size - off,
2374 "\t%hhu. Port %hhu, Partition %hhu\n",
2375 idx, ndev->peers[idx].port, ndev->peers[idx].part);
2379 data = idt_ntb_link_is_up(&ndev->ntb, &speed, &width);
2380 off += scnprintf(strbuf + off, size - off,
2381 "NTB link status\t- 0x%08x, ", data);
2382 off += scnprintf(strbuf + off, size - off, "PCIe Gen %d x%d lanes\n",
2385 /* Mapping table entries */
2386 off += scnprintf(strbuf + off, size - off, "NTB Mapping Table:\n");
2387 for (idx = 0; idx < IDT_MTBL_ENTRY_CNT; idx++) {
2388 spin_lock_irqsave(&ndev->mtbl_lock, irqflags);
2389 idt_nt_write(ndev, IDT_NT_NTMTBLADDR, idx);
2390 data = idt_nt_read(ndev, IDT_NT_NTMTBLDATA);
2391 spin_unlock_irqrestore(&ndev->mtbl_lock, irqflags);
2393 /* Print valid entries only */
2394 if (data & IDT_NTMTBLDATA_VALID) {
2395 off += scnprintf(strbuf + off, size - off,
2396 "\t%hhu. Partition %d, Requester ID 0x%04x\n",
2397 idx, GET_FIELD(NTMTBLDATA_PART, data),
2398 GET_FIELD(NTMTBLDATA_REQID, data));
2401 off += scnprintf(strbuf + off, size - off, "\n");
2403 /* Outbound memory windows information */
2404 off += scnprintf(strbuf + off, size - off,
2405 "Outbound Memory Windows:\n");
2406 for (idx = 0; idx < ndev->mw_cnt; idx += cnt) {
2407 data = ndev->mws[idx].type;
2408 cnt = idt_get_mw_count(data);
2410 /* Print Memory Window information */
2411 if (data == IDT_MW_DIR)
2412 off += scnprintf(strbuf + off, size - off,
2415 off += scnprintf(strbuf + off, size - off,
2416 "\t%hhu-%hhu.\t", idx, idx + cnt - 1);
2418 off += scnprintf(strbuf + off, size - off, "%s BAR%hhu, ",
2419 idt_get_mw_name(data), ndev->mws[idx].bar);
2421 off += scnprintf(strbuf + off, size - off,
2422 "Address align 0x%08llx, ", ndev->mws[idx].addr_align);
2424 off += scnprintf(strbuf + off, size - off,
2425 "Size align 0x%08llx, Size max %llu\n",
2426 ndev->mws[idx].size_align, ndev->mws[idx].size_max);
2429 /* Inbound memory windows information */
2430 for (pidx = 0; pidx < ndev->peer_cnt; pidx++) {
2431 off += scnprintf(strbuf + off, size - off,
2432 "Inbound Memory Windows for peer %hhu (Port %hhu):\n",
2433 pidx, ndev->peers[pidx].port);
2435 /* Print Memory Windows information */
2436 for (idx = 0; idx < ndev->peers[pidx].mw_cnt; idx += cnt) {
2437 data = ndev->peers[pidx].mws[idx].type;
2438 cnt = idt_get_mw_count(data);
2440 if (data == IDT_MW_DIR)
2441 off += scnprintf(strbuf + off, size - off,
2444 off += scnprintf(strbuf + off, size - off,
2445 "\t%hhu-%hhu.\t", idx, idx + cnt - 1);
2447 off += scnprintf(strbuf + off, size - off,
2448 "%s BAR%hhu, ", idt_get_mw_name(data),
2449 ndev->peers[pidx].mws[idx].bar);
2451 off += scnprintf(strbuf + off, size - off,
2452 "Address align 0x%08llx, ",
2453 ndev->peers[pidx].mws[idx].addr_align);
2455 off += scnprintf(strbuf + off, size - off,
2456 "Size align 0x%08llx, Size max %llu\n",
2457 ndev->peers[pidx].mws[idx].size_align,
2458 ndev->peers[pidx].mws[idx].size_max);
2461 off += scnprintf(strbuf + off, size - off, "\n");
2463 /* Doorbell information */
2464 data = idt_sw_read(ndev, IDT_SW_GDBELLSTS);
2465 off += scnprintf(strbuf + off, size - off,
2466 "Global Doorbell state\t- 0x%08x\n", data);
2467 data = idt_ntb_db_read(&ndev->ntb);
2468 off += scnprintf(strbuf + off, size - off,
2469 "Local Doorbell state\t- 0x%08x\n", data);
2470 data = idt_nt_read(ndev, IDT_NT_INDBELLMSK);
2471 off += scnprintf(strbuf + off, size - off,
2472 "Local Doorbell mask\t- 0x%08x\n", data);
2473 off += scnprintf(strbuf + off, size - off, "\n");
2475 /* Messaging information */
2476 off += scnprintf(strbuf + off, size - off,
2477 "Message event valid\t- 0x%08x\n", IDT_MSG_MASK);
2478 data = idt_ntb_msg_read_sts(&ndev->ntb);
2479 off += scnprintf(strbuf + off, size - off,
2480 "Message event status\t- 0x%08x\n", data);
2481 data = idt_nt_read(ndev, IDT_NT_MSGSTSMSK);
2482 off += scnprintf(strbuf + off, size - off,
2483 "Message event mask\t- 0x%08x\n", data);
2484 off += scnprintf(strbuf + off, size - off,
2486 for (idx = 0; idx < IDT_MSG_CNT; idx++) {
2488 data = idt_ntb_msg_read(&ndev->ntb, &src, idx);
2489 off += scnprintf(strbuf + off, size - off,
2490 "\t%hhu. 0x%08x from peer %hhu (Port %hhu)\n",
2491 idx, data, src, ndev->peers[src].port);
2493 off += scnprintf(strbuf + off, size - off, "\n");
2495 /* Current temperature */
2496 idt_read_temp(ndev, IDT_TEMP_CUR, &mdeg);
2497 off += scnprintf(strbuf + off, size - off,
2498 "Switch temperature\t\t- %hhd.%hhuC\n",
2499 idt_get_deg(mdeg), idt_get_deg_frac(mdeg));
2501 /* Copy the buffer to the User Space */
2502 ret = simple_read_from_buffer(ubuf, count, offp, strbuf, off);
2509 * idt_init_dbgfs() - initialize DebugFS node
2510 * @ndev: IDT NTB hardware driver descriptor
2512 * Return: zero on success, otherwise a negative error number.
2514 static int idt_init_dbgfs(struct idt_ntb_dev *ndev)
2518 /* If the top directory is not created then do nothing */
2519 if (IS_ERR_OR_NULL(dbgfs_topdir)) {
2520 dev_info(&ndev->ntb.pdev->dev, "Top DebugFS directory absent");
2521 return PTR_ERR(dbgfs_topdir);
2524 /* Create the info file node */
2525 snprintf(devname, 64, "info:%s", pci_name(ndev->ntb.pdev));
2526 ndev->dbgfs_info = debugfs_create_file(devname, 0400, dbgfs_topdir,
2527 ndev, &idt_dbgfs_info_ops);
2528 if (IS_ERR(ndev->dbgfs_info)) {
2529 dev_dbg(&ndev->ntb.pdev->dev, "Failed to create DebugFS node");
2530 return PTR_ERR(ndev->dbgfs_info);
2533 dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node created");
2539 * idt_deinit_dbgfs() - deinitialize DebugFS node
2540 * @ndev: IDT NTB hardware driver descriptor
2542 * Just discard the info node from DebugFS
2544 static void idt_deinit_dbgfs(struct idt_ntb_dev *ndev)
2546 debugfs_remove(ndev->dbgfs_info);
2548 dev_dbg(&ndev->ntb.pdev->dev, "NTB device DebugFS node discarded");
2551 /*=============================================================================
2552 * 11. Basic PCIe device initialization
2553 *=============================================================================
2557 * idt_check_setup() - Check whether the IDT PCIe-swtich is properly
2559 * @pdev: Pointer to the PCI device descriptor
2561 * Return: zero on success, otherwise a negative error number.
2563 static int idt_check_setup(struct pci_dev *pdev)
2568 /* Read the BARSETUP0 */
2569 ret = pci_read_config_dword(pdev, IDT_NT_BARSETUP0, &data);
2572 "Failed to read BARSETUP0 config register");
2576 /* Check whether the BAR0 register is enabled to be of config space */
2577 if (!(data & IDT_BARSETUP_EN) || !(data & IDT_BARSETUP_MODE_CFG)) {
2578 dev_err(&pdev->dev, "BAR0 doesn't map config space");
2582 /* Configuration space BAR0 must have certain size */
2583 if ((data & IDT_BARSETUP_SIZE_MASK) != IDT_BARSETUP_SIZE_CFG) {
2584 dev_err(&pdev->dev, "Invalid size of config space");
2588 dev_dbg(&pdev->dev, "NTB device pre-initialized correctly");
2594 * Create the IDT PCIe-switch driver descriptor
2595 * @pdev: Pointer to the PCI device descriptor
2596 * @id: IDT PCIe-device configuration
2598 * It just allocates a memory for IDT PCIe-switch device structure and
2599 * initializes some commonly used fields.
2601 * No need of release method, since managed device resource is used for
2602 * memory allocation.
2604 * Return: pointer to the descriptor, otherwise a negative error number.
2606 static struct idt_ntb_dev *idt_create_dev(struct pci_dev *pdev,
2607 const struct pci_device_id *id)
2609 struct idt_ntb_dev *ndev;
2611 /* Allocate memory for the IDT PCIe-device descriptor */
2612 ndev = devm_kzalloc(&pdev->dev, sizeof(*ndev), GFP_KERNEL);
2614 dev_err(&pdev->dev, "Memory allocation failed for descriptor");
2615 return ERR_PTR(-ENOMEM);
2618 /* Save the IDT PCIe-switch ports configuration */
2619 ndev->swcfg = (struct idt_89hpes_cfg *)id->driver_data;
2620 /* Save the PCI-device pointer inside the NTB device structure */
2621 ndev->ntb.pdev = pdev;
2623 /* Initialize spin locker of Doorbell, Message and GASA registers */
2624 spin_lock_init(&ndev->db_mask_lock);
2625 spin_lock_init(&ndev->msg_mask_lock);
2626 spin_lock_init(&ndev->gasa_lock);
2628 dev_info(&pdev->dev, "IDT %s discovered", ndev->swcfg->name);
2630 dev_dbg(&pdev->dev, "NTB device descriptor created");
2636 * idt_init_pci() - initialize the basic PCI-related subsystem
2637 * @ndev: Pointer to the IDT PCIe-switch driver descriptor
2639 * Managed device resources will be freed automatically in case of failure or
2640 * driver detachment.
2642 * Return: zero on success, otherwise negative error number.
2644 static int idt_init_pci(struct idt_ntb_dev *ndev)
2646 struct pci_dev *pdev = ndev->ntb.pdev;
2649 /* Initialize the bit mask of PCI/NTB DMA */
2650 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(64));
2652 ret = pci_set_dma_mask(pdev, DMA_BIT_MASK(32));
2654 dev_err(&pdev->dev, "Failed to set DMA bit mask\n");
2657 dev_warn(&pdev->dev, "Cannot set DMA highmem bit mask\n");
2659 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(64));
2661 ret = pci_set_consistent_dma_mask(pdev, DMA_BIT_MASK(32));
2664 "Failed to set consistent DMA bit mask\n");
2667 dev_warn(&pdev->dev,
2668 "Cannot set consistent DMA highmem bit mask\n");
2670 ret = dma_coerce_mask_and_coherent(&ndev->ntb.dev,
2671 dma_get_mask(&pdev->dev));
2673 dev_err(&pdev->dev, "Failed to set NTB device DMA bit mask\n");
2678 * Enable the device advanced error reporting. It's not critical to
2679 * have AER disabled in the kernel.
2681 ret = pci_enable_pcie_error_reporting(pdev);
2683 dev_warn(&pdev->dev, "PCIe AER capability disabled\n");
2684 else /* Cleanup uncorrectable error status before getting to init */
2685 pci_cleanup_aer_uncorrect_error_status(pdev);
2687 /* First enable the PCI device */
2688 ret = pcim_enable_device(pdev);
2690 dev_err(&pdev->dev, "Failed to enable PCIe device\n");
2691 goto err_disable_aer;
2695 * Enable the bus mastering, which effectively enables MSI IRQs and
2696 * Request TLPs translation
2698 pci_set_master(pdev);
2700 /* Request all BARs resources and map BAR0 only */
2701 ret = pcim_iomap_regions_request_all(pdev, 1, NTB_NAME);
2703 dev_err(&pdev->dev, "Failed to request resources\n");
2704 goto err_clear_master;
2707 /* Retrieve virtual address of BAR0 - PCI configuration space */
2708 ndev->cfgspc = pcim_iomap_table(pdev)[0];
2710 /* Put the IDT driver data pointer to the PCI-device private pointer */
2711 pci_set_drvdata(pdev, ndev);
2713 dev_dbg(&pdev->dev, "NT-function PCIe interface initialized");
2718 pci_clear_master(pdev);
2720 (void)pci_disable_pcie_error_reporting(pdev);
2726 * idt_deinit_pci() - deinitialize the basic PCI-related subsystem
2727 * @ndev: Pointer to the IDT PCIe-switch driver descriptor
2729 * Managed resources will be freed on the driver detachment
2731 static void idt_deinit_pci(struct idt_ntb_dev *ndev)
2733 struct pci_dev *pdev = ndev->ntb.pdev;
2735 /* Clean up the PCI-device private data pointer */
2736 pci_set_drvdata(pdev, NULL);
2738 /* Clear the bus master disabling the Request TLPs translation */
2739 pci_clear_master(pdev);
2741 /* Disable the AER capability */
2742 (void)pci_disable_pcie_error_reporting(pdev);
2744 dev_dbg(&pdev->dev, "NT-function PCIe interface cleared");
2747 /*===========================================================================
2748 * 12. PCI bus callback functions
2749 *===========================================================================
2753 * idt_pci_probe() - PCI device probe callback
2754 * @pdev: Pointer to PCI device structure
2755 * @id: PCIe device custom descriptor
2757 * Return: zero on success, otherwise negative error number
2759 static int idt_pci_probe(struct pci_dev *pdev,
2760 const struct pci_device_id *id)
2762 struct idt_ntb_dev *ndev;
2765 /* Check whether IDT PCIe-switch is properly pre-initialized */
2766 ret = idt_check_setup(pdev);
2770 /* Allocate the memory for IDT NTB device data */
2771 ndev = idt_create_dev(pdev, id);
2772 if (IS_ERR_OR_NULL(ndev))
2773 return PTR_ERR(ndev);
2775 /* Initialize the basic PCI subsystem of the device */
2776 ret = idt_init_pci(ndev);
2780 /* Scan ports of the IDT PCIe-switch */
2781 (void)idt_scan_ports(ndev);
2783 /* Initialize NTB link events subsystem */
2784 idt_init_link(ndev);
2786 /* Initialize MWs subsystem */
2787 ret = idt_init_mws(ndev);
2789 goto err_deinit_link;
2791 /* Initialize Messaging subsystem */
2794 /* Initialize hwmon interface */
2795 idt_init_temp(ndev);
2797 /* Initialize IDT interrupts handler */
2798 ret = idt_init_isr(ndev);
2800 goto err_deinit_link;
2802 /* Register IDT NTB devices on the NTB bus */
2803 ret = idt_register_device(ndev);
2805 goto err_deinit_isr;
2807 /* Initialize DebugFS info node */
2808 (void)idt_init_dbgfs(ndev);
2810 /* IDT PCIe-switch NTB driver is finally initialized */
2811 dev_info(&pdev->dev, "IDT NTB device is ready");
2813 /* May the force be with us... */
2817 idt_deinit_isr(ndev);
2819 idt_deinit_link(ndev);
2820 idt_deinit_pci(ndev);
2826 * idt_pci_probe() - PCI device remove callback
2827 * @pdev: Pointer to PCI device structure
2829 static void idt_pci_remove(struct pci_dev *pdev)
2831 struct idt_ntb_dev *ndev = pci_get_drvdata(pdev);
2833 /* Deinit the DebugFS node */
2834 idt_deinit_dbgfs(ndev);
2836 /* Unregister NTB device */
2837 idt_unregister_device(ndev);
2839 /* Stop the interrupts handling */
2840 idt_deinit_isr(ndev);
2842 /* Deinitialize link event subsystem */
2843 idt_deinit_link(ndev);
2845 /* Deinit basic PCI subsystem */
2846 idt_deinit_pci(ndev);
2848 /* IDT PCIe-switch NTB driver is finally initialized */
2849 dev_info(&pdev->dev, "IDT NTB device is removed");
2855 * IDT PCIe-switch models ports configuration structures
2857 static const struct idt_89hpes_cfg idt_89hpes24nt6ag2_config = {
2858 .name = "89HPES24NT6AG2",
2859 .port_cnt = 6, .ports = {0, 2, 4, 6, 8, 12}
2861 static const struct idt_89hpes_cfg idt_89hpes32nt8ag2_config = {
2862 .name = "89HPES32NT8AG2",
2863 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2865 static const struct idt_89hpes_cfg idt_89hpes32nt8bg2_config = {
2866 .name = "89HPES32NT8BG2",
2867 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2869 static const struct idt_89hpes_cfg idt_89hpes12nt12g2_config = {
2870 .name = "89HPES12NT12G2",
2871 .port_cnt = 3, .ports = {0, 8, 16}
2873 static const struct idt_89hpes_cfg idt_89hpes16nt16g2_config = {
2874 .name = "89HPES16NT16G2",
2875 .port_cnt = 4, .ports = {0, 8, 12, 16}
2877 static const struct idt_89hpes_cfg idt_89hpes24nt24g2_config = {
2878 .name = "89HPES24NT24G2",
2879 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2881 static const struct idt_89hpes_cfg idt_89hpes32nt24ag2_config = {
2882 .name = "89HPES32NT24AG2",
2883 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2885 static const struct idt_89hpes_cfg idt_89hpes32nt24bg2_config = {
2886 .name = "89HPES32NT24BG2",
2887 .port_cnt = 8, .ports = {0, 2, 4, 6, 8, 12, 16, 20}
2891 * PCI-ids table of the supported IDT PCIe-switch devices
2893 static const struct pci_device_id idt_pci_tbl[] = {
2894 {IDT_PCI_DEVICE_IDS(89HPES24NT6AG2, idt_89hpes24nt6ag2_config)},
2895 {IDT_PCI_DEVICE_IDS(89HPES32NT8AG2, idt_89hpes32nt8ag2_config)},
2896 {IDT_PCI_DEVICE_IDS(89HPES32NT8BG2, idt_89hpes32nt8bg2_config)},
2897 {IDT_PCI_DEVICE_IDS(89HPES12NT12G2, idt_89hpes12nt12g2_config)},
2898 {IDT_PCI_DEVICE_IDS(89HPES16NT16G2, idt_89hpes16nt16g2_config)},
2899 {IDT_PCI_DEVICE_IDS(89HPES24NT24G2, idt_89hpes24nt24g2_config)},
2900 {IDT_PCI_DEVICE_IDS(89HPES32NT24AG2, idt_89hpes32nt24ag2_config)},
2901 {IDT_PCI_DEVICE_IDS(89HPES32NT24BG2, idt_89hpes32nt24bg2_config)},
2904 MODULE_DEVICE_TABLE(pci, idt_pci_tbl);
2907 * IDT PCIe-switch NT-function device driver structure definition
2909 static struct pci_driver idt_pci_driver = {
2910 .name = KBUILD_MODNAME,
2911 .probe = idt_pci_probe,
2912 .remove = idt_pci_remove,
2913 .id_table = idt_pci_tbl,
2916 static int __init idt_pci_driver_init(void)
2918 pr_info("%s %s\n", NTB_DESC, NTB_VER);
2920 /* Create the top DebugFS directory if the FS is initialized */
2921 if (debugfs_initialized())
2922 dbgfs_topdir = debugfs_create_dir(KBUILD_MODNAME, NULL);
2924 /* Register the NTB hardware driver to handle the PCI device */
2925 return pci_register_driver(&idt_pci_driver);
2927 module_init(idt_pci_driver_init);
2929 static void __exit idt_pci_driver_exit(void)
2931 /* Unregister the NTB hardware driver */
2932 pci_unregister_driver(&idt_pci_driver);
2934 /* Discard the top DebugFS directory */
2935 debugfs_remove_recursive(dbgfs_topdir);
2937 module_exit(idt_pci_driver_exit);