1 // SPDX-License-Identifier: GPL-2.0
3 * Copyright (C) 2017 Marvell
5 * Antoine Tenart <antoine.tenart@free-electrons.com>
9 #include <linux/device.h>
10 #include <linux/dma-mapping.h>
11 #include <linux/dmapool.h>
12 #include <linux/firmware.h>
13 #include <linux/interrupt.h>
14 #include <linux/module.h>
15 #include <linux/of_platform.h>
16 #include <linux/of_irq.h>
17 #include <linux/pci.h>
18 #include <linux/platform_device.h>
19 #include <linux/workqueue.h>
21 #include <crypto/internal/aead.h>
22 #include <crypto/internal/hash.h>
23 #include <crypto/internal/skcipher.h>
27 static u32 max_rings = EIP197_MAX_RINGS;
28 module_param(max_rings, uint, 0644);
29 MODULE_PARM_DESC(max_rings, "Maximum number of rings to use.");
31 static void eip197_trc_cache_setupvirt(struct safexcel_crypto_priv *priv)
36 * Map all interfaces/rings to register index 0
37 * so they can share contexts. Without this, the EIP197 will
38 * assume each interface/ring to be in its own memory domain
39 * i.e. have its own subset of UNIQUE memory addresses.
40 * Which would cause records with the SAME memory address to
41 * use DIFFERENT cache buffers, causing both poor cache utilization
42 * AND serious coherence/invalidation issues.
44 for (i = 0; i < 4; i++)
45 writel(0, priv->base + EIP197_FLUE_IFC_LUT(i));
48 * Initialize other virtualization regs for cache
49 * These may not be in their reset state ...
51 for (i = 0; i < priv->config.rings; i++) {
52 writel(0, priv->base + EIP197_FLUE_CACHEBASE_LO(i));
53 writel(0, priv->base + EIP197_FLUE_CACHEBASE_HI(i));
54 writel(EIP197_FLUE_CONFIG_MAGIC,
55 priv->base + EIP197_FLUE_CONFIG(i));
57 writel(0, priv->base + EIP197_FLUE_OFFSETS);
58 writel(0, priv->base + EIP197_FLUE_ARC4_OFFSET);
61 static void eip197_trc_cache_banksel(struct safexcel_crypto_priv *priv,
62 u32 addrmid, int *actbank)
67 curbank = addrmid >> 16;
68 if (curbank != *actbank) {
69 val = readl(priv->base + EIP197_CS_RAM_CTRL);
70 val = (val & ~EIP197_CS_BANKSEL_MASK) |
71 (curbank << EIP197_CS_BANKSEL_OFS);
72 writel(val, priv->base + EIP197_CS_RAM_CTRL);
77 static u32 eip197_trc_cache_probe(struct safexcel_crypto_priv *priv,
78 int maxbanks, u32 probemask)
80 u32 val, addrhi, addrlo, addrmid;
84 * And probe the actual size of the physically attached cache data RAM
85 * Using a binary subdivision algorithm downto 32 byte cache lines.
87 addrhi = 1 << (16 + maxbanks);
89 actbank = min(maxbanks - 1, 0);
90 while ((addrhi - addrlo) > 32) {
91 /* write marker to lowest address in top half */
92 addrmid = (addrhi + addrlo) >> 1;
93 eip197_trc_cache_banksel(priv, addrmid, &actbank);
94 writel((addrmid | (addrlo << 16)) & probemask,
95 priv->base + EIP197_CLASSIFICATION_RAMS +
98 /* write marker to lowest address in bottom half */
99 eip197_trc_cache_banksel(priv, addrlo, &actbank);
100 writel((addrlo | (addrhi << 16)) & probemask,
101 priv->base + EIP197_CLASSIFICATION_RAMS +
104 /* read back marker from top half */
105 eip197_trc_cache_banksel(priv, addrmid, &actbank);
106 val = readl(priv->base + EIP197_CLASSIFICATION_RAMS +
109 if (val == ((addrmid | (addrlo << 16)) & probemask)) {
110 /* read back correct, continue with top half */
113 /* not read back correct, continue with bottom half */
120 static void eip197_trc_cache_clear(struct safexcel_crypto_priv *priv,
121 int cs_rc_max, int cs_ht_wc)
124 u32 htable_offset, val, offset;
126 /* Clear all records in administration RAM */
127 for (i = 0; i < cs_rc_max; i++) {
128 offset = EIP197_CLASSIFICATION_RAMS + i * EIP197_CS_RC_SIZE;
130 writel(EIP197_CS_RC_NEXT(EIP197_RC_NULL) |
131 EIP197_CS_RC_PREV(EIP197_RC_NULL),
132 priv->base + offset);
134 val = EIP197_CS_RC_NEXT(i + 1) | EIP197_CS_RC_PREV(i - 1);
136 val |= EIP197_CS_RC_PREV(EIP197_RC_NULL);
137 else if (i == cs_rc_max - 1)
138 val |= EIP197_CS_RC_NEXT(EIP197_RC_NULL);
139 writel(val, priv->base + offset + 4);
140 /* must also initialize the address key due to ECC! */
141 writel(0, priv->base + offset + 8);
142 writel(0, priv->base + offset + 12);
145 /* Clear the hash table entries */
146 htable_offset = cs_rc_max * EIP197_CS_RC_SIZE;
147 for (i = 0; i < cs_ht_wc; i++)
148 writel(GENMASK(29, 0),
149 priv->base + EIP197_CLASSIFICATION_RAMS +
150 htable_offset + i * sizeof(u32));
153 static void eip197_trc_cache_init(struct safexcel_crypto_priv *priv)
155 u32 val, dsize, asize;
156 int cs_rc_max, cs_ht_wc, cs_trc_rec_wc, cs_trc_lg_rec_wc;
157 int cs_rc_abs_max, cs_ht_sz;
160 /* Setup (dummy) virtualization for cache */
161 eip197_trc_cache_setupvirt(priv);
164 * Enable the record cache memory access and
165 * probe the bank select width
167 val = readl(priv->base + EIP197_CS_RAM_CTRL);
168 val &= ~EIP197_TRC_ENABLE_MASK;
169 val |= EIP197_TRC_ENABLE_0 | EIP197_CS_BANKSEL_MASK;
170 writel(val, priv->base + EIP197_CS_RAM_CTRL);
171 val = readl(priv->base + EIP197_CS_RAM_CTRL);
172 maxbanks = ((val&EIP197_CS_BANKSEL_MASK)>>EIP197_CS_BANKSEL_OFS) + 1;
174 /* Clear all ECC errors */
175 writel(0, priv->base + EIP197_TRC_ECCCTRL);
178 * Make sure the cache memory is accessible by taking record cache into
179 * reset. Need data memory access here, not admin access.
181 val = readl(priv->base + EIP197_TRC_PARAMS);
182 val |= EIP197_TRC_PARAMS_SW_RESET | EIP197_TRC_PARAMS_DATA_ACCESS;
183 writel(val, priv->base + EIP197_TRC_PARAMS);
185 /* Probed data RAM size in bytes */
186 dsize = eip197_trc_cache_probe(priv, maxbanks, 0xffffffff);
189 * Now probe the administration RAM size pretty much the same way
190 * Except that only the lower 30 bits are writable and we don't need
193 val = readl(priv->base + EIP197_TRC_PARAMS);
194 /* admin access now */
195 val &= ~(EIP197_TRC_PARAMS_DATA_ACCESS | EIP197_CS_BANKSEL_MASK);
196 writel(val, priv->base + EIP197_TRC_PARAMS);
198 /* Probed admin RAM size in admin words */
199 asize = eip197_trc_cache_probe(priv, 0, 0xbfffffff) >> 4;
201 /* Clear any ECC errors detected while probing! */
202 writel(0, priv->base + EIP197_TRC_ECCCTRL);
205 * Determine optimal configuration from RAM sizes
206 * Note that we assume that the physical RAM configuration is sane
207 * Therefore, we don't do any parameter error checking here ...
210 /* For now, just use a single record format covering everything */
211 cs_trc_rec_wc = EIP197_CS_TRC_REC_WC;
212 cs_trc_lg_rec_wc = EIP197_CS_TRC_REC_WC;
215 * Step #1: How many records will physically fit?
216 * Hard upper limit is 1023!
218 cs_rc_abs_max = min_t(uint, ((dsize >> 2) / cs_trc_lg_rec_wc), 1023);
219 /* Step #2: Need at least 2 words in the admin RAM per record */
220 cs_rc_max = min_t(uint, cs_rc_abs_max, (asize >> 1));
221 /* Step #3: Determine log2 of hash table size */
222 cs_ht_sz = __fls(asize - cs_rc_max) - 2;
223 /* Step #4: determine current size of hash table in dwords */
224 cs_ht_wc = 16<<cs_ht_sz; /* dwords, not admin words */
225 /* Step #5: add back excess words and see if we can fit more records */
226 cs_rc_max = min_t(uint, cs_rc_abs_max, asize - (cs_ht_wc >> 4));
228 /* Clear the cache RAMs */
229 eip197_trc_cache_clear(priv, cs_rc_max, cs_ht_wc);
231 /* Disable the record cache memory access */
232 val = readl(priv->base + EIP197_CS_RAM_CTRL);
233 val &= ~EIP197_TRC_ENABLE_MASK;
234 writel(val, priv->base + EIP197_CS_RAM_CTRL);
236 /* Write head and tail pointers of the record free chain */
237 val = EIP197_TRC_FREECHAIN_HEAD_PTR(0) |
238 EIP197_TRC_FREECHAIN_TAIL_PTR(cs_rc_max - 1);
239 writel(val, priv->base + EIP197_TRC_FREECHAIN);
241 /* Configure the record cache #1 */
242 val = EIP197_TRC_PARAMS2_RC_SZ_SMALL(cs_trc_rec_wc) |
243 EIP197_TRC_PARAMS2_HTABLE_PTR(cs_rc_max);
244 writel(val, priv->base + EIP197_TRC_PARAMS2);
246 /* Configure the record cache #2 */
247 val = EIP197_TRC_PARAMS_RC_SZ_LARGE(cs_trc_lg_rec_wc) |
248 EIP197_TRC_PARAMS_BLK_TIMER_SPEED(1) |
249 EIP197_TRC_PARAMS_HTABLE_SZ(cs_ht_sz);
250 writel(val, priv->base + EIP197_TRC_PARAMS);
252 dev_info(priv->dev, "TRC init: %dd,%da (%dr,%dh)\n",
253 dsize, asize, cs_rc_max, cs_ht_wc + cs_ht_wc);
256 static void eip197_init_firmware(struct safexcel_crypto_priv *priv)
261 for (pe = 0; pe < priv->config.pes; pe++) {
262 /* Configure the token FIFO's */
263 writel(3, EIP197_PE(priv) + EIP197_PE_ICE_PUTF_CTRL(pe));
264 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_PPTF_CTRL(pe));
266 /* Clear the ICE scratchpad memory */
267 val = readl(EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
268 val |= EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_TIMER |
269 EIP197_PE_ICE_SCRATCH_CTRL_TIMER_EN |
270 EIP197_PE_ICE_SCRATCH_CTRL_SCRATCH_ACCESS |
271 EIP197_PE_ICE_SCRATCH_CTRL_CHANGE_ACCESS;
272 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_SCRATCH_CTRL(pe));
274 /* clear the scratchpad RAM using 32 bit writes only */
275 for (i = 0; i < EIP197_NUM_OF_SCRATCH_BLOCKS; i++)
276 writel(0, EIP197_PE(priv) +
277 EIP197_PE_ICE_SCRATCH_RAM(pe) + (i << 2));
279 /* Reset the IFPP engine to make its program mem accessible */
280 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
281 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
282 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
283 EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
285 /* Reset the IPUE engine to make its program mem accessible */
286 writel(EIP197_PE_ICE_x_CTRL_SW_RESET |
287 EIP197_PE_ICE_x_CTRL_CLR_ECC_CORR |
288 EIP197_PE_ICE_x_CTRL_CLR_ECC_NON_CORR,
289 EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
291 /* Enable access to all IFPP program memories */
292 writel(EIP197_PE_ICE_RAM_CTRL_FPP_PROG_EN,
293 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
298 static int eip197_write_firmware(struct safexcel_crypto_priv *priv,
299 const struct firmware *fw)
301 const u32 *data = (const u32 *)fw->data;
304 /* Write the firmware */
305 for (i = 0; i < fw->size / sizeof(u32); i++)
306 writel(be32_to_cpu(data[i]),
307 priv->base + EIP197_CLASSIFICATION_RAMS + i * sizeof(u32));
309 /* Exclude final 2 NOPs from size */
310 return i - EIP197_FW_TERMINAL_NOPS;
314 * If FW is actual production firmware, then poll for its initialization
315 * to complete and check if it is good for the HW, otherwise just return OK.
317 static bool poll_fw_ready(struct safexcel_crypto_priv *priv, int fpp)
323 pollofs = EIP197_FW_FPP_READY;
325 pollofs = EIP197_FW_PUE_READY;
327 for (pe = 0; pe < priv->config.pes; pe++) {
328 base = EIP197_PE_ICE_SCRATCH_RAM(pe);
329 pollcnt = EIP197_FW_START_POLLCNT;
331 (readl_relaxed(EIP197_PE(priv) + base +
336 dev_err(priv->dev, "FW(%d) for PE %d failed to start\n",
344 static bool eip197_start_firmware(struct safexcel_crypto_priv *priv,
345 int ipuesz, int ifppsz, int minifw)
350 for (pe = 0; pe < priv->config.pes; pe++) {
351 /* Disable access to all program memory */
352 writel(0, EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
354 /* Start IFPP microengines */
358 val = EIP197_PE_ICE_UENG_START_OFFSET((ifppsz - 1) &
359 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
360 EIP197_PE_ICE_UENG_DEBUG_RESET;
361 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_FPP_CTRL(pe));
363 /* Start IPUE microengines */
367 val = EIP197_PE_ICE_UENG_START_OFFSET((ipuesz - 1) &
368 EIP197_PE_ICE_UENG_INIT_ALIGN_MASK) |
369 EIP197_PE_ICE_UENG_DEBUG_RESET;
370 writel(val, EIP197_PE(priv) + EIP197_PE_ICE_PUE_CTRL(pe));
373 /* For miniFW startup, there is no initialization, so always succeed */
377 /* Wait until all the firmwares have properly started up */
378 if (!poll_fw_ready(priv, 1))
380 if (!poll_fw_ready(priv, 0))
386 static int eip197_load_firmwares(struct safexcel_crypto_priv *priv)
388 const char *fw_name[] = {"ifpp.bin", "ipue.bin"};
389 const struct firmware *fw[FW_NB];
390 char fw_path[37], *dir = NULL;
391 int i, j, ret = 0, pe;
392 int ipuesz, ifppsz, minifw = 0;
394 if (priv->version == EIP197D_MRVL)
396 else if (priv->version == EIP197B_MRVL ||
397 priv->version == EIP197_DEVBRD)
403 for (i = 0; i < FW_NB; i++) {
404 snprintf(fw_path, 37, "inside-secure/%s/%s", dir, fw_name[i]);
405 ret = firmware_request_nowarn(&fw[i], fw_path, priv->dev);
407 if (minifw || priv->version != EIP197B_MRVL)
410 /* Fallback to the old firmware location for the
413 ret = firmware_request_nowarn(&fw[i], fw_name[i],
420 eip197_init_firmware(priv);
422 ifppsz = eip197_write_firmware(priv, fw[FW_IFPP]);
424 /* Enable access to IPUE program memories */
425 for (pe = 0; pe < priv->config.pes; pe++)
426 writel(EIP197_PE_ICE_RAM_CTRL_PUE_PROG_EN,
427 EIP197_PE(priv) + EIP197_PE_ICE_RAM_CTRL(pe));
429 ipuesz = eip197_write_firmware(priv, fw[FW_IPUE]);
431 if (eip197_start_firmware(priv, ipuesz, ifppsz, minifw)) {
432 dev_dbg(priv->dev, "Firmware loaded successfully\n");
439 for (j = 0; j < i; j++)
440 release_firmware(fw[j]);
443 /* Retry with minifw path */
444 dev_dbg(priv->dev, "Firmware set not (fully) present or init failed, falling back to BCLA mode\n");
445 dir = "eip197_minifw";
450 dev_dbg(priv->dev, "Firmware load failed.\n");
455 static int safexcel_hw_setup_cdesc_rings(struct safexcel_crypto_priv *priv)
457 u32 cd_size_rnd, val;
460 cd_size_rnd = (priv->config.cd_size +
461 (BIT(priv->hwconfig.hwdataw) - 1)) >>
462 priv->hwconfig.hwdataw;
463 /* determine number of CD's we can fetch into the CD FIFO as 1 block */
464 if (priv->flags & SAFEXCEL_HW_EIP197) {
465 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
466 cd_fetch_cnt = (1 << priv->hwconfig.hwcfsize) / cd_size_rnd;
467 cd_fetch_cnt = min_t(uint, cd_fetch_cnt,
468 (priv->config.pes * EIP197_FETCH_DEPTH));
470 /* for the EIP97, just fetch all that fits minus 1 */
471 cd_fetch_cnt = ((1 << priv->hwconfig.hwcfsize) /
475 for (i = 0; i < priv->config.rings; i++) {
476 /* ring base address */
477 writel(lower_32_bits(priv->ring[i].cdr.base_dma),
478 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
479 writel(upper_32_bits(priv->ring[i].cdr.base_dma),
480 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
482 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.cd_offset << 16) |
483 priv->config.cd_size,
484 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
485 writel(((cd_fetch_cnt *
486 (cd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
487 (cd_fetch_cnt * priv->config.cd_offset),
488 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
490 /* Configure DMA tx control */
491 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
492 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
493 writel(val, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
495 /* clear any pending interrupt */
496 writel(GENMASK(5, 0),
497 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
503 static int safexcel_hw_setup_rdesc_rings(struct safexcel_crypto_priv *priv)
505 u32 rd_size_rnd, val;
508 /* determine number of RD's we can fetch into the FIFO as one block */
509 rd_size_rnd = (EIP197_RD64_FETCH_SIZE +
510 (BIT(priv->hwconfig.hwdataw) - 1)) >>
511 priv->hwconfig.hwdataw;
512 if (priv->flags & SAFEXCEL_HW_EIP197) {
513 /* EIP197: try to fetch enough in 1 go to keep all pipes busy */
514 rd_fetch_cnt = (1 << priv->hwconfig.hwrfsize) / rd_size_rnd;
515 rd_fetch_cnt = min_t(uint, rd_fetch_cnt,
516 (priv->config.pes * EIP197_FETCH_DEPTH));
518 /* for the EIP97, just fetch all that fits minus 1 */
519 rd_fetch_cnt = ((1 << priv->hwconfig.hwrfsize) /
523 for (i = 0; i < priv->config.rings; i++) {
524 /* ring base address */
525 writel(lower_32_bits(priv->ring[i].rdr.base_dma),
526 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
527 writel(upper_32_bits(priv->ring[i].rdr.base_dma),
528 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
530 writel(EIP197_xDR_DESC_MODE_64BIT | (priv->config.rd_offset << 16) |
531 priv->config.rd_size,
532 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DESC_SIZE);
534 writel(((rd_fetch_cnt *
535 (rd_size_rnd << priv->hwconfig.hwdataw)) << 16) |
536 (rd_fetch_cnt * priv->config.rd_offset),
537 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
539 /* Configure DMA tx control */
540 val = EIP197_HIA_xDR_CFG_WR_CACHE(WR_CACHE_3BITS);
541 val |= EIP197_HIA_xDR_CFG_RD_CACHE(RD_CACHE_3BITS);
542 val |= EIP197_HIA_xDR_WR_RES_BUF | EIP197_HIA_xDR_WR_CTRL_BUF;
544 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_DMA_CFG);
546 /* clear any pending interrupt */
547 writel(GENMASK(7, 0),
548 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
550 /* enable ring interrupt */
551 val = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
552 val |= EIP197_RDR_IRQ(i);
553 writel(val, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CTRL(i));
559 static int safexcel_hw_init(struct safexcel_crypto_priv *priv)
564 dev_dbg(priv->dev, "HW init: using %d pipe(s) and %d ring(s)\n",
565 priv->config.pes, priv->config.rings);
568 * For EIP197's only set maximum number of TX commands to 2^5 = 32
569 * Skip for the EIP97 as it does not have this field.
571 if (priv->flags & SAFEXCEL_HW_EIP197) {
572 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
573 val |= EIP197_MST_CTRL_TX_MAX_CMD(5);
574 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
577 /* Configure wr/rd cache values */
578 writel(EIP197_MST_CTRL_RD_CACHE(RD_CACHE_4BITS) |
579 EIP197_MST_CTRL_WD_CACHE(WR_CACHE_4BITS),
580 EIP197_HIA_GEN_CFG(priv) + EIP197_MST_CTRL);
582 /* Interrupts reset */
584 /* Disable all global interrupts */
585 writel(0, EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ENABLE_CTRL);
587 /* Clear any pending interrupt */
588 writel(GENMASK(31, 0), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
590 /* Processing Engine configuration */
591 for (pe = 0; pe < priv->config.pes; pe++) {
592 /* Data Fetch Engine configuration */
594 /* Reset all DFE threads */
595 writel(EIP197_DxE_THR_CTRL_RESET_PE,
596 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
598 if (priv->flags & SAFEXCEL_HW_EIP197)
599 /* Reset HIA input interface arbiter (EIP197 only) */
600 writel(EIP197_HIA_RA_PE_CTRL_RESET,
601 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
603 /* DMA transfer size to use */
604 val = EIP197_HIA_DFE_CFG_DIS_DEBUG;
605 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(6) |
606 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(9);
607 val |= EIP197_HIA_DxE_CFG_MIN_CTRL_SIZE(6) |
608 EIP197_HIA_DxE_CFG_MAX_CTRL_SIZE(7);
609 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(RD_CACHE_3BITS);
610 val |= EIP197_HIA_DxE_CFG_CTRL_CACHE_CTRL(RD_CACHE_3BITS);
611 writel(val, EIP197_HIA_DFE(priv) + EIP197_HIA_DFE_CFG(pe));
613 /* Leave the DFE threads reset state */
614 writel(0, EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
616 /* Configure the processing engine thresholds */
617 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
618 EIP197_PE_IN_xBUF_THRES_MAX(9),
619 EIP197_PE(priv) + EIP197_PE_IN_DBUF_THRES(pe));
620 writel(EIP197_PE_IN_xBUF_THRES_MIN(6) |
621 EIP197_PE_IN_xBUF_THRES_MAX(7),
622 EIP197_PE(priv) + EIP197_PE_IN_TBUF_THRES(pe));
624 if (priv->flags & SAFEXCEL_HW_EIP197)
625 /* enable HIA input interface arbiter and rings */
626 writel(EIP197_HIA_RA_PE_CTRL_EN |
627 GENMASK(priv->config.rings - 1, 0),
628 EIP197_HIA_AIC(priv) + EIP197_HIA_RA_PE_CTRL(pe));
630 /* Data Store Engine configuration */
632 /* Reset all DSE threads */
633 writel(EIP197_DxE_THR_CTRL_RESET_PE,
634 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
636 /* Wait for all DSE threads to complete */
637 while ((readl(EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_STAT(pe)) &
638 GENMASK(15, 12)) != GENMASK(15, 12))
641 /* DMA transfer size to use */
642 val = EIP197_HIA_DSE_CFG_DIS_DEBUG;
643 val |= EIP197_HIA_DxE_CFG_MIN_DATA_SIZE(7) |
644 EIP197_HIA_DxE_CFG_MAX_DATA_SIZE(8);
645 val |= EIP197_HIA_DxE_CFG_DATA_CACHE_CTRL(WR_CACHE_3BITS);
646 val |= EIP197_HIA_DSE_CFG_ALWAYS_BUFFERABLE;
647 /* FIXME: instability issues can occur for EIP97 but disabling
648 * it impacts performance.
650 if (priv->flags & SAFEXCEL_HW_EIP197)
651 val |= EIP197_HIA_DSE_CFG_EN_SINGLE_WR;
652 writel(val, EIP197_HIA_DSE(priv) + EIP197_HIA_DSE_CFG(pe));
654 /* Leave the DSE threads reset state */
655 writel(0, EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
657 /* Configure the procesing engine thresholds */
658 writel(EIP197_PE_OUT_DBUF_THRES_MIN(7) |
659 EIP197_PE_OUT_DBUF_THRES_MAX(8),
660 EIP197_PE(priv) + EIP197_PE_OUT_DBUF_THRES(pe));
662 /* Processing Engine configuration */
664 /* Token & context configuration */
665 val = EIP197_PE_EIP96_TOKEN_CTRL_CTX_UPDATES |
666 EIP197_PE_EIP96_TOKEN_CTRL_NO_TOKEN_WAIT |
667 EIP197_PE_EIP96_TOKEN_CTRL_ENABLE_TIMEOUT;
668 writel(val, EIP197_PE(priv) + EIP197_PE_EIP96_TOKEN_CTRL(pe));
670 /* H/W capabilities selection: just enable everything */
671 writel(EIP197_FUNCTION_ALL,
672 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION_EN(pe));
673 writel(EIP197_FUNCTION_ALL,
674 EIP197_PE(priv) + EIP197_PE_EIP96_FUNCTION2_EN(pe));
677 /* Command Descriptor Rings prepare */
678 for (i = 0; i < priv->config.rings; i++) {
679 /* Clear interrupts for this ring */
680 writel(GENMASK(31, 0),
681 EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLE_CLR(i));
683 /* Disable external triggering */
684 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_CFG);
686 /* Clear the pending prepared counter */
687 writel(EIP197_xDR_PREP_CLR_COUNT,
688 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
690 /* Clear the pending processed counter */
691 writel(EIP197_xDR_PROC_CLR_COUNT,
692 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
695 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
697 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
699 writel((EIP197_DEFAULT_RING_SIZE * priv->config.cd_offset) << 2,
700 EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
703 /* Result Descriptor Ring prepare */
704 for (i = 0; i < priv->config.rings; i++) {
705 /* Disable external triggering*/
706 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_CFG);
708 /* Clear the pending prepared counter */
709 writel(EIP197_xDR_PREP_CLR_COUNT,
710 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_COUNT);
712 /* Clear the pending processed counter */
713 writel(EIP197_xDR_PROC_CLR_COUNT,
714 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_COUNT);
717 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PREP_PNTR);
719 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_PROC_PNTR);
722 writel((EIP197_DEFAULT_RING_SIZE * priv->config.rd_offset) << 2,
723 EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_SIZE);
726 for (pe = 0; pe < priv->config.pes; pe++) {
727 /* Enable command descriptor rings */
728 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
729 EIP197_HIA_DFE_THR(priv) + EIP197_HIA_DFE_THR_CTRL(pe));
731 /* Enable result descriptor rings */
732 writel(EIP197_DxE_THR_CTRL_EN | GENMASK(priv->config.rings - 1, 0),
733 EIP197_HIA_DSE_THR(priv) + EIP197_HIA_DSE_THR_CTRL(pe));
736 /* Clear any HIA interrupt */
737 writel(GENMASK(30, 20), EIP197_HIA_AIC_G(priv) + EIP197_HIA_AIC_G_ACK);
739 if (priv->flags & SAFEXCEL_HW_EIP197) {
740 eip197_trc_cache_init(priv);
741 priv->flags |= EIP197_TRC_CACHE;
743 ret = eip197_load_firmwares(priv);
748 safexcel_hw_setup_cdesc_rings(priv);
749 safexcel_hw_setup_rdesc_rings(priv);
754 /* Called with ring's lock taken */
755 static void safexcel_try_push_requests(struct safexcel_crypto_priv *priv,
758 int coal = min_t(int, priv->ring[ring].requests, EIP197_MAX_BATCH_SZ);
763 /* Configure when we want an interrupt */
764 writel(EIP197_HIA_RDR_THRESH_PKT_MODE |
765 EIP197_HIA_RDR_THRESH_PROC_PKT(coal),
766 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_THRESH);
769 void safexcel_dequeue(struct safexcel_crypto_priv *priv, int ring)
771 struct crypto_async_request *req, *backlog;
772 struct safexcel_context *ctx;
773 int ret, nreq = 0, cdesc = 0, rdesc = 0, commands, results;
775 /* If a request wasn't properly dequeued because of a lack of resources,
776 * proceeded it first,
778 req = priv->ring[ring].req;
779 backlog = priv->ring[ring].backlog;
784 spin_lock_bh(&priv->ring[ring].queue_lock);
785 backlog = crypto_get_backlog(&priv->ring[ring].queue);
786 req = crypto_dequeue_request(&priv->ring[ring].queue);
787 spin_unlock_bh(&priv->ring[ring].queue_lock);
790 priv->ring[ring].req = NULL;
791 priv->ring[ring].backlog = NULL;
796 ctx = crypto_tfm_ctx(req->tfm);
797 ret = ctx->send(req, ring, &commands, &results);
802 backlog->complete(backlog, -EINPROGRESS);
804 /* In case the send() helper did not issue any command to push
805 * to the engine because the input data was cached, continue to
806 * dequeue other requests as this is valid and not an error.
808 if (!commands && !results)
817 /* Not enough resources to handle all the requests. Bail out and save
818 * the request and the backlog for the next dequeue call (per-ring).
820 priv->ring[ring].req = req;
821 priv->ring[ring].backlog = backlog;
827 spin_lock_bh(&priv->ring[ring].lock);
829 priv->ring[ring].requests += nreq;
831 if (!priv->ring[ring].busy) {
832 safexcel_try_push_requests(priv, ring);
833 priv->ring[ring].busy = true;
836 spin_unlock_bh(&priv->ring[ring].lock);
838 /* let the RDR know we have pending descriptors */
839 writel((rdesc * priv->config.rd_offset) << 2,
840 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
842 /* let the CDR know we have pending descriptors */
843 writel((cdesc * priv->config.cd_offset) << 2,
844 EIP197_HIA_CDR(priv, ring) + EIP197_HIA_xDR_PREP_COUNT);
847 inline int safexcel_rdesc_check_errors(struct safexcel_crypto_priv *priv,
848 struct safexcel_result_desc *rdesc)
850 if (likely((!rdesc->descriptor_overflow) &&
851 (!rdesc->buffer_overflow) &&
852 (!rdesc->result_data.error_code)))
855 if (rdesc->descriptor_overflow)
856 dev_err(priv->dev, "Descriptor overflow detected");
858 if (rdesc->buffer_overflow)
859 dev_err(priv->dev, "Buffer overflow detected");
861 if (rdesc->result_data.error_code & 0x4066) {
862 /* Fatal error (bits 1,2,5,6 & 14) */
864 "result descriptor error (%x)",
865 rdesc->result_data.error_code);
867 } else if (rdesc->result_data.error_code &
868 (BIT(7) | BIT(4) | BIT(3) | BIT(0))) {
870 * Give priority over authentication fails:
871 * Blocksize, length & overflow errors,
872 * something wrong with the input!
875 } else if (rdesc->result_data.error_code & BIT(9)) {
876 /* Authentication failed */
880 /* All other non-fatal errors */
884 inline void safexcel_rdr_req_set(struct safexcel_crypto_priv *priv,
886 struct safexcel_result_desc *rdesc,
887 struct crypto_async_request *req)
889 int i = safexcel_ring_rdr_rdesc_index(priv, ring, rdesc);
891 priv->ring[ring].rdr_req[i] = req;
894 inline struct crypto_async_request *
895 safexcel_rdr_req_get(struct safexcel_crypto_priv *priv, int ring)
897 int i = safexcel_ring_first_rdr_index(priv, ring);
899 return priv->ring[ring].rdr_req[i];
902 void safexcel_complete(struct safexcel_crypto_priv *priv, int ring)
904 struct safexcel_command_desc *cdesc;
906 /* Acknowledge the command descriptors */
908 cdesc = safexcel_ring_next_rptr(priv, &priv->ring[ring].cdr);
911 "Could not retrieve the command descriptor\n");
914 } while (!cdesc->last_seg);
917 void safexcel_inv_complete(struct crypto_async_request *req, int error)
919 struct safexcel_inv_result *result = req->data;
921 if (error == -EINPROGRESS)
924 result->error = error;
925 complete(&result->completion);
928 int safexcel_invalidate_cache(struct crypto_async_request *async,
929 struct safexcel_crypto_priv *priv,
930 dma_addr_t ctxr_dma, int ring)
932 struct safexcel_command_desc *cdesc;
933 struct safexcel_result_desc *rdesc;
936 /* Prepare command descriptor */
937 cdesc = safexcel_add_cdesc(priv, ring, true, true, 0, 0, 0, ctxr_dma);
939 return PTR_ERR(cdesc);
941 cdesc->control_data.type = EIP197_TYPE_EXTENDED;
942 cdesc->control_data.options = 0;
943 cdesc->control_data.refresh = 0;
944 cdesc->control_data.control0 = CONTEXT_CONTROL_INV_TR;
946 /* Prepare result descriptor */
947 rdesc = safexcel_add_rdesc(priv, ring, true, true, 0, 0);
950 ret = PTR_ERR(rdesc);
954 safexcel_rdr_req_set(priv, ring, rdesc, async);
959 safexcel_ring_rollback_wptr(priv, &priv->ring[ring].cdr);
964 static inline void safexcel_handle_result_descriptor(struct safexcel_crypto_priv *priv,
967 struct crypto_async_request *req;
968 struct safexcel_context *ctx;
969 int ret, i, nreq, ndesc, tot_descs, handled = 0;
970 bool should_complete;
975 nreq = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
976 nreq >>= EIP197_xDR_PROC_xD_PKT_OFFSET;
977 nreq &= EIP197_xDR_PROC_xD_PKT_MASK;
981 for (i = 0; i < nreq; i++) {
982 req = safexcel_rdr_req_get(priv, ring);
984 ctx = crypto_tfm_ctx(req->tfm);
985 ndesc = ctx->handle_result(priv, ring, req,
986 &should_complete, &ret);
988 dev_err(priv->dev, "failed to handle result (%d)\n",
993 if (should_complete) {
995 req->complete(req, ret);
1005 writel(EIP197_xDR_PROC_xD_PKT(i) |
1006 EIP197_xDR_PROC_xD_COUNT(tot_descs * priv->config.rd_offset),
1007 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_PROC_COUNT);
1009 /* If the number of requests overflowed the counter, try to proceed more
1012 if (nreq == EIP197_xDR_PROC_xD_PKT_MASK)
1013 goto handle_results;
1016 spin_lock_bh(&priv->ring[ring].lock);
1018 priv->ring[ring].requests -= handled;
1019 safexcel_try_push_requests(priv, ring);
1021 if (!priv->ring[ring].requests)
1022 priv->ring[ring].busy = false;
1024 spin_unlock_bh(&priv->ring[ring].lock);
1027 static void safexcel_dequeue_work(struct work_struct *work)
1029 struct safexcel_work_data *data =
1030 container_of(work, struct safexcel_work_data, work);
1032 safexcel_dequeue(data->priv, data->ring);
1035 struct safexcel_ring_irq_data {
1036 struct safexcel_crypto_priv *priv;
1040 static irqreturn_t safexcel_irq_ring(int irq, void *data)
1042 struct safexcel_ring_irq_data *irq_data = data;
1043 struct safexcel_crypto_priv *priv = irq_data->priv;
1044 int ring = irq_data->ring, rc = IRQ_NONE;
1047 status = readl(EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ENABLED_STAT(ring));
1051 /* RDR interrupts */
1052 if (status & EIP197_RDR_IRQ(ring)) {
1053 stat = readl(EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1055 if (unlikely(stat & EIP197_xDR_ERR)) {
1057 * Fatal error, the RDR is unusable and must be
1058 * reinitialized. This should not happen under
1059 * normal circumstances.
1061 dev_err(priv->dev, "RDR: fatal error.\n");
1062 } else if (likely(stat & EIP197_xDR_THRESH)) {
1063 rc = IRQ_WAKE_THREAD;
1066 /* ACK the interrupts */
1068 EIP197_HIA_RDR(priv, ring) + EIP197_HIA_xDR_STAT);
1071 /* ACK the interrupts */
1072 writel(status, EIP197_HIA_AIC_R(priv) + EIP197_HIA_AIC_R_ACK(ring));
1077 static irqreturn_t safexcel_irq_ring_thread(int irq, void *data)
1079 struct safexcel_ring_irq_data *irq_data = data;
1080 struct safexcel_crypto_priv *priv = irq_data->priv;
1081 int ring = irq_data->ring;
1083 safexcel_handle_result_descriptor(priv, ring);
1085 queue_work(priv->ring[ring].workqueue,
1086 &priv->ring[ring].work_data.work);
1091 static int safexcel_request_ring_irq(void *pdev, int irqid,
1093 irq_handler_t handler,
1094 irq_handler_t threaded_handler,
1095 struct safexcel_ring_irq_data *ring_irq_priv)
1100 if (IS_ENABLED(CONFIG_PCI) && is_pci_dev) {
1101 struct pci_dev *pci_pdev = pdev;
1103 dev = &pci_pdev->dev;
1104 irq = pci_irq_vector(pci_pdev, irqid);
1106 dev_err(dev, "unable to get device MSI IRQ %d (err %d)\n",
1110 } else if (IS_ENABLED(CONFIG_OF)) {
1111 struct platform_device *plf_pdev = pdev;
1112 char irq_name[6] = {0}; /* "ringX\0" */
1114 snprintf(irq_name, 6, "ring%d", irqid);
1115 dev = &plf_pdev->dev;
1116 irq = platform_get_irq_byname(plf_pdev, irq_name);
1119 dev_err(dev, "unable to get IRQ '%s' (err %d)\n",
1125 ret = devm_request_threaded_irq(dev, irq, handler,
1126 threaded_handler, IRQF_ONESHOT,
1127 dev_name(dev), ring_irq_priv);
1129 dev_err(dev, "unable to request IRQ %d\n", irq);
1136 static struct safexcel_alg_template *safexcel_algs[] = {
1137 &safexcel_alg_ecb_des,
1138 &safexcel_alg_cbc_des,
1139 &safexcel_alg_ecb_des3_ede,
1140 &safexcel_alg_cbc_des3_ede,
1141 &safexcel_alg_ecb_aes,
1142 &safexcel_alg_cbc_aes,
1143 &safexcel_alg_cfb_aes,
1144 &safexcel_alg_ofb_aes,
1145 &safexcel_alg_ctr_aes,
1148 &safexcel_alg_sha224,
1149 &safexcel_alg_sha256,
1150 &safexcel_alg_sha384,
1151 &safexcel_alg_sha512,
1152 &safexcel_alg_hmac_md5,
1153 &safexcel_alg_hmac_sha1,
1154 &safexcel_alg_hmac_sha224,
1155 &safexcel_alg_hmac_sha256,
1156 &safexcel_alg_hmac_sha384,
1157 &safexcel_alg_hmac_sha512,
1158 &safexcel_alg_authenc_hmac_sha1_cbc_aes,
1159 &safexcel_alg_authenc_hmac_sha224_cbc_aes,
1160 &safexcel_alg_authenc_hmac_sha256_cbc_aes,
1161 &safexcel_alg_authenc_hmac_sha384_cbc_aes,
1162 &safexcel_alg_authenc_hmac_sha512_cbc_aes,
1163 &safexcel_alg_authenc_hmac_sha1_cbc_des3_ede,
1164 &safexcel_alg_authenc_hmac_sha1_ctr_aes,
1165 &safexcel_alg_authenc_hmac_sha224_ctr_aes,
1166 &safexcel_alg_authenc_hmac_sha256_ctr_aes,
1167 &safexcel_alg_authenc_hmac_sha384_ctr_aes,
1168 &safexcel_alg_authenc_hmac_sha512_ctr_aes,
1169 &safexcel_alg_xts_aes,
1174 static int safexcel_register_algorithms(struct safexcel_crypto_priv *priv)
1178 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1179 safexcel_algs[i]->priv = priv;
1181 /* Do we have all required base algorithms available? */
1182 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1183 safexcel_algs[i]->algo_mask)
1184 /* No, so don't register this ciphersuite */
1187 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1188 ret = crypto_register_skcipher(&safexcel_algs[i]->alg.skcipher);
1189 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1190 ret = crypto_register_aead(&safexcel_algs[i]->alg.aead);
1192 ret = crypto_register_ahash(&safexcel_algs[i]->alg.ahash);
1201 for (j = 0; j < i; j++) {
1202 /* Do we have all required base algorithms available? */
1203 if ((safexcel_algs[j]->algo_mask & priv->hwconfig.algo_flags) !=
1204 safexcel_algs[j]->algo_mask)
1205 /* No, so don't unregister this ciphersuite */
1208 if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1209 crypto_unregister_skcipher(&safexcel_algs[j]->alg.skcipher);
1210 else if (safexcel_algs[j]->type == SAFEXCEL_ALG_TYPE_AEAD)
1211 crypto_unregister_aead(&safexcel_algs[j]->alg.aead);
1213 crypto_unregister_ahash(&safexcel_algs[j]->alg.ahash);
1219 static void safexcel_unregister_algorithms(struct safexcel_crypto_priv *priv)
1223 for (i = 0; i < ARRAY_SIZE(safexcel_algs); i++) {
1224 /* Do we have all required base algorithms available? */
1225 if ((safexcel_algs[i]->algo_mask & priv->hwconfig.algo_flags) !=
1226 safexcel_algs[i]->algo_mask)
1227 /* No, so don't unregister this ciphersuite */
1230 if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_SKCIPHER)
1231 crypto_unregister_skcipher(&safexcel_algs[i]->alg.skcipher);
1232 else if (safexcel_algs[i]->type == SAFEXCEL_ALG_TYPE_AEAD)
1233 crypto_unregister_aead(&safexcel_algs[i]->alg.aead);
1235 crypto_unregister_ahash(&safexcel_algs[i]->alg.ahash);
1239 static void safexcel_configure(struct safexcel_crypto_priv *priv)
1243 val = readl(EIP197_HIA_AIC_G(priv) + EIP197_HIA_OPTIONS);
1245 /* Read number of PEs from the engine */
1246 if (priv->flags & SAFEXCEL_HW_EIP197)
1247 /* Wider field width for all EIP197 type engines */
1248 mask = EIP197_N_PES_MASK;
1250 /* Narrow field width for EIP97 type engine */
1251 mask = EIP97_N_PES_MASK;
1253 priv->config.pes = (val >> EIP197_N_PES_OFFSET) & mask;
1255 priv->config.rings = min_t(u32, val & GENMASK(3, 0), max_rings);
1257 val = (val & GENMASK(27, 25)) >> 25;
1258 mask = BIT(val) - 1;
1260 priv->config.cd_size = (sizeof(struct safexcel_command_desc) / sizeof(u32));
1261 priv->config.cd_offset = (priv->config.cd_size + mask) & ~mask;
1263 priv->config.rd_size = (sizeof(struct safexcel_result_desc) / sizeof(u32));
1264 priv->config.rd_offset = (priv->config.rd_size + mask) & ~mask;
1267 static void safexcel_init_register_offsets(struct safexcel_crypto_priv *priv)
1269 struct safexcel_register_offsets *offsets = &priv->offsets;
1271 if (priv->flags & SAFEXCEL_HW_EIP197) {
1272 offsets->hia_aic = EIP197_HIA_AIC_BASE;
1273 offsets->hia_aic_g = EIP197_HIA_AIC_G_BASE;
1274 offsets->hia_aic_r = EIP197_HIA_AIC_R_BASE;
1275 offsets->hia_aic_xdr = EIP197_HIA_AIC_xDR_BASE;
1276 offsets->hia_dfe = EIP197_HIA_DFE_BASE;
1277 offsets->hia_dfe_thr = EIP197_HIA_DFE_THR_BASE;
1278 offsets->hia_dse = EIP197_HIA_DSE_BASE;
1279 offsets->hia_dse_thr = EIP197_HIA_DSE_THR_BASE;
1280 offsets->hia_gen_cfg = EIP197_HIA_GEN_CFG_BASE;
1281 offsets->pe = EIP197_PE_BASE;
1282 offsets->global = EIP197_GLOBAL_BASE;
1284 offsets->hia_aic = EIP97_HIA_AIC_BASE;
1285 offsets->hia_aic_g = EIP97_HIA_AIC_G_BASE;
1286 offsets->hia_aic_r = EIP97_HIA_AIC_R_BASE;
1287 offsets->hia_aic_xdr = EIP97_HIA_AIC_xDR_BASE;
1288 offsets->hia_dfe = EIP97_HIA_DFE_BASE;
1289 offsets->hia_dfe_thr = EIP97_HIA_DFE_THR_BASE;
1290 offsets->hia_dse = EIP97_HIA_DSE_BASE;
1291 offsets->hia_dse_thr = EIP97_HIA_DSE_THR_BASE;
1292 offsets->hia_gen_cfg = EIP97_HIA_GEN_CFG_BASE;
1293 offsets->pe = EIP97_PE_BASE;
1294 offsets->global = EIP97_GLOBAL_BASE;
1299 * Generic part of probe routine, shared by platform and PCI driver
1301 * Assumes IO resources have been mapped, private data mem has been allocated,
1302 * clocks have been enabled, device pointer has been assigned etc.
1305 static int safexcel_probe_generic(void *pdev,
1306 struct safexcel_crypto_priv *priv,
1309 struct device *dev = priv->dev;
1310 u32 peid, version, mask, val, hiaopt;
1313 priv->context_pool = dmam_pool_create("safexcel-context", dev,
1314 sizeof(struct safexcel_context_record),
1316 if (!priv->context_pool)
1320 * First try the EIP97 HIA version regs
1321 * For the EIP197, this is guaranteed to NOT return any of the test
1324 version = readl(priv->base + EIP97_HIA_AIC_BASE + EIP197_HIA_VERSION);
1326 mask = 0; /* do not swap */
1327 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1328 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1329 } else if (EIP197_REG_HI16(version) == EIP197_HIA_VERSION_BE) {
1330 /* read back byte-swapped, so complement byte swap bits */
1331 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1332 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1334 /* So it wasn't an EIP97 ... maybe it's an EIP197? */
1335 version = readl(priv->base + EIP197_HIA_AIC_BASE +
1336 EIP197_HIA_VERSION);
1337 if (EIP197_REG_LO16(version) == EIP197_HIA_VERSION_LE) {
1338 priv->hwconfig.hiaver = EIP197_VERSION_MASK(version);
1339 priv->flags |= SAFEXCEL_HW_EIP197;
1340 } else if (EIP197_REG_HI16(version) ==
1341 EIP197_HIA_VERSION_BE) {
1342 /* read back byte-swapped, so complement swap bits */
1343 mask = EIP197_MST_CTRL_BYTE_SWAP_BITS;
1344 priv->hwconfig.hiaver = EIP197_VERSION_SWAP(version);
1345 priv->flags |= SAFEXCEL_HW_EIP197;
1351 /* Now initialize the reg offsets based on the probing info so far */
1352 safexcel_init_register_offsets(priv);
1355 * If the version was read byte-swapped, we need to flip the device
1356 * swapping Keep in mind here, though, that what we write will also be
1360 val = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1361 val = val ^ (mask >> 24); /* toggle byte swap bits */
1362 writel(val, EIP197_HIA_AIC(priv) + EIP197_HIA_MST_CTRL);
1366 * We're not done probing yet! We may fall through to here if no HIA
1367 * was found at all. So, with the endianness presumably correct now and
1368 * the offsets setup, *really* probe for the EIP97/EIP197.
1370 version = readl(EIP197_GLOBAL(priv) + EIP197_VERSION);
1371 if (((priv->flags & SAFEXCEL_HW_EIP197) &&
1372 (EIP197_REG_LO16(version) != EIP197_VERSION_LE)) ||
1373 ((!(priv->flags & SAFEXCEL_HW_EIP197) &&
1374 (EIP197_REG_LO16(version) != EIP97_VERSION_LE)))) {
1376 * We did not find the device that matched our initial probing
1377 * (or our initial probing failed) Report appropriate error.
1382 priv->hwconfig.hwver = EIP197_VERSION_MASK(version);
1383 hwctg = version >> 28;
1384 peid = version & 255;
1386 /* Detect EIP96 packet engine and version */
1387 version = readl(EIP197_PE(priv) + EIP197_PE_EIP96_VERSION(0));
1388 if (EIP197_REG_LO16(version) != EIP96_VERSION_LE) {
1389 dev_err(dev, "EIP%d: EIP96 not detected.\n", peid);
1392 priv->hwconfig.pever = EIP197_VERSION_MASK(version);
1394 hiaopt = readl(EIP197_HIA_AIC(priv) + EIP197_HIA_OPTIONS);
1396 if (priv->flags & SAFEXCEL_HW_EIP197) {
1398 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1399 EIP197_HWDATAW_MASK;
1400 priv->hwconfig.hwcfsize = ((hiaopt >> EIP197_CFSIZE_OFFSET) &
1401 EIP197_CFSIZE_MASK) +
1402 EIP197_CFSIZE_ADJUST;
1403 priv->hwconfig.hwrfsize = ((hiaopt >> EIP197_RFSIZE_OFFSET) &
1404 EIP197_RFSIZE_MASK) +
1405 EIP197_RFSIZE_ADJUST;
1408 priv->hwconfig.hwdataw = (hiaopt >> EIP197_HWDATAW_OFFSET) &
1410 priv->hwconfig.hwcfsize = (hiaopt >> EIP97_CFSIZE_OFFSET) &
1412 priv->hwconfig.hwrfsize = (hiaopt >> EIP97_RFSIZE_OFFSET) &
1416 /* Get supported algorithms from EIP96 transform engine */
1417 priv->hwconfig.algo_flags = readl(EIP197_PE(priv) +
1418 EIP197_PE_EIP96_OPTIONS(0));
1420 /* Print single info line describing what we just detected */
1421 dev_info(priv->dev, "EIP%d:%x(%d)-HIA:%x(%d,%d,%d),PE:%x,alg:%08x\n",
1422 peid, priv->hwconfig.hwver, hwctg, priv->hwconfig.hiaver,
1423 priv->hwconfig.hwdataw, priv->hwconfig.hwcfsize,
1424 priv->hwconfig.hwrfsize, priv->hwconfig.pever,
1425 priv->hwconfig.algo_flags);
1427 safexcel_configure(priv);
1429 if (IS_ENABLED(CONFIG_PCI) && priv->version == EIP197_DEVBRD) {
1431 * Request MSI vectors for global + 1 per ring -
1432 * or just 1 for older dev images
1434 struct pci_dev *pci_pdev = pdev;
1436 ret = pci_alloc_irq_vectors(pci_pdev,
1437 priv->config.rings + 1,
1438 priv->config.rings + 1,
1439 PCI_IRQ_MSI | PCI_IRQ_MSIX);
1441 dev_err(dev, "Failed to allocate PCI MSI interrupts\n");
1446 /* Register the ring IRQ handlers and configure the rings */
1447 priv->ring = devm_kcalloc(dev, priv->config.rings,
1448 sizeof(*priv->ring),
1453 for (i = 0; i < priv->config.rings; i++) {
1454 char wq_name[9] = {0};
1456 struct safexcel_ring_irq_data *ring_irq;
1458 ret = safexcel_init_ring_descriptors(priv,
1460 &priv->ring[i].rdr);
1462 dev_err(dev, "Failed to initialize rings\n");
1466 priv->ring[i].rdr_req = devm_kcalloc(dev,
1467 EIP197_DEFAULT_RING_SIZE,
1468 sizeof(priv->ring[i].rdr_req),
1470 if (!priv->ring[i].rdr_req)
1473 ring_irq = devm_kzalloc(dev, sizeof(*ring_irq), GFP_KERNEL);
1477 ring_irq->priv = priv;
1480 irq = safexcel_request_ring_irq(pdev,
1481 EIP197_IRQ_NUMBER(i, is_pci_dev),
1484 safexcel_irq_ring_thread,
1487 dev_err(dev, "Failed to get IRQ ID for ring %d\n", i);
1491 priv->ring[i].work_data.priv = priv;
1492 priv->ring[i].work_data.ring = i;
1493 INIT_WORK(&priv->ring[i].work_data.work,
1494 safexcel_dequeue_work);
1496 snprintf(wq_name, 9, "wq_ring%d", i);
1497 priv->ring[i].workqueue =
1498 create_singlethread_workqueue(wq_name);
1499 if (!priv->ring[i].workqueue)
1502 priv->ring[i].requests = 0;
1503 priv->ring[i].busy = false;
1505 crypto_init_queue(&priv->ring[i].queue,
1506 EIP197_DEFAULT_RING_SIZE);
1508 spin_lock_init(&priv->ring[i].lock);
1509 spin_lock_init(&priv->ring[i].queue_lock);
1512 atomic_set(&priv->ring_used, 0);
1514 ret = safexcel_hw_init(priv);
1516 dev_err(dev, "HW init failed (%d)\n", ret);
1520 ret = safexcel_register_algorithms(priv);
1522 dev_err(dev, "Failed to register algorithms (%d)\n", ret);
1529 static void safexcel_hw_reset_rings(struct safexcel_crypto_priv *priv)
1533 for (i = 0; i < priv->config.rings; i++) {
1534 /* clear any pending interrupt */
1535 writel(GENMASK(5, 0), EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_STAT);
1536 writel(GENMASK(7, 0), EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_STAT);
1538 /* Reset the CDR base address */
1539 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1540 writel(0, EIP197_HIA_CDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1542 /* Reset the RDR base address */
1543 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_LO);
1544 writel(0, EIP197_HIA_RDR(priv, i) + EIP197_HIA_xDR_RING_BASE_ADDR_HI);
1548 #if IS_ENABLED(CONFIG_OF)
1549 /* for Device Tree platform driver */
1551 static int safexcel_probe(struct platform_device *pdev)
1553 struct device *dev = &pdev->dev;
1554 struct safexcel_crypto_priv *priv;
1557 priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL);
1562 priv->version = (enum safexcel_eip_version)of_device_get_match_data(dev);
1564 platform_set_drvdata(pdev, priv);
1566 priv->base = devm_platform_ioremap_resource(pdev, 0);
1567 if (IS_ERR(priv->base)) {
1568 dev_err(dev, "failed to get resource\n");
1569 return PTR_ERR(priv->base);
1572 priv->clk = devm_clk_get(&pdev->dev, NULL);
1573 ret = PTR_ERR_OR_ZERO(priv->clk);
1574 /* The clock isn't mandatory */
1575 if (ret != -ENOENT) {
1579 ret = clk_prepare_enable(priv->clk);
1581 dev_err(dev, "unable to enable clk (%d)\n", ret);
1586 priv->reg_clk = devm_clk_get(&pdev->dev, "reg");
1587 ret = PTR_ERR_OR_ZERO(priv->reg_clk);
1588 /* The clock isn't mandatory */
1589 if (ret != -ENOENT) {
1593 ret = clk_prepare_enable(priv->reg_clk);
1595 dev_err(dev, "unable to enable reg clk (%d)\n", ret);
1600 ret = dma_set_mask_and_coherent(dev, DMA_BIT_MASK(64));
1604 /* Generic EIP97/EIP197 device probing */
1605 ret = safexcel_probe_generic(pdev, priv, 0);
1612 clk_disable_unprepare(priv->reg_clk);
1614 clk_disable_unprepare(priv->clk);
1618 static int safexcel_remove(struct platform_device *pdev)
1620 struct safexcel_crypto_priv *priv = platform_get_drvdata(pdev);
1623 safexcel_unregister_algorithms(priv);
1624 safexcel_hw_reset_rings(priv);
1626 clk_disable_unprepare(priv->clk);
1628 for (i = 0; i < priv->config.rings; i++)
1629 destroy_workqueue(priv->ring[i].workqueue);
1634 static const struct of_device_id safexcel_of_match_table[] = {
1636 .compatible = "inside-secure,safexcel-eip97ies",
1637 .data = (void *)EIP97IES_MRVL,
1640 .compatible = "inside-secure,safexcel-eip197b",
1641 .data = (void *)EIP197B_MRVL,
1644 .compatible = "inside-secure,safexcel-eip197d",
1645 .data = (void *)EIP197D_MRVL,
1647 /* For backward compatibility and intended for generic use */
1649 .compatible = "inside-secure,safexcel-eip97",
1650 .data = (void *)EIP97IES_MRVL,
1653 .compatible = "inside-secure,safexcel-eip197",
1654 .data = (void *)EIP197B_MRVL,
1659 static struct platform_driver crypto_safexcel = {
1660 .probe = safexcel_probe,
1661 .remove = safexcel_remove,
1663 .name = "crypto-safexcel",
1664 .of_match_table = safexcel_of_match_table,
1669 #if IS_ENABLED(CONFIG_PCI)
1670 /* PCIE devices - i.e. Inside Secure development boards */
1672 static int safexcel_pci_probe(struct pci_dev *pdev,
1673 const struct pci_device_id *ent)
1675 struct device *dev = &pdev->dev;
1676 struct safexcel_crypto_priv *priv;
1677 void __iomem *pciebase;
1681 dev_dbg(dev, "Probing PCIE device: vendor %04x, device %04x, subv %04x, subdev %04x, ctxt %lx\n",
1682 ent->vendor, ent->device, ent->subvendor,
1683 ent->subdevice, ent->driver_data);
1685 priv = kzalloc(sizeof(*priv), GFP_KERNEL);
1690 priv->version = (enum safexcel_eip_version)ent->driver_data;
1692 pci_set_drvdata(pdev, priv);
1694 /* enable the device */
1695 rc = pcim_enable_device(pdev);
1697 dev_err(dev, "Failed to enable PCI device\n");
1701 /* take ownership of PCI BAR0 */
1702 rc = pcim_iomap_regions(pdev, 1, "crypto_safexcel");
1704 dev_err(dev, "Failed to map IO region for BAR0\n");
1707 priv->base = pcim_iomap_table(pdev)[0];
1709 if (priv->version == EIP197_DEVBRD) {
1710 dev_dbg(dev, "Device identified as FPGA based development board - applying HW reset\n");
1712 rc = pcim_iomap_regions(pdev, 4, "crypto_safexcel");
1714 dev_err(dev, "Failed to map IO region for BAR4\n");
1718 pciebase = pcim_iomap_table(pdev)[2];
1719 val = readl(pciebase + EIP197_XLX_IRQ_BLOCK_ID_ADDR);
1720 if ((val >> 16) == EIP197_XLX_IRQ_BLOCK_ID_VALUE) {
1721 dev_dbg(dev, "Detected Xilinx PCIE IRQ block version %d, multiple MSI support enabled\n",
1724 /* Setup MSI identity map mapping */
1725 writel(EIP197_XLX_USER_VECT_LUT0_IDENT,
1726 pciebase + EIP197_XLX_USER_VECT_LUT0_ADDR);
1727 writel(EIP197_XLX_USER_VECT_LUT1_IDENT,
1728 pciebase + EIP197_XLX_USER_VECT_LUT1_ADDR);
1729 writel(EIP197_XLX_USER_VECT_LUT2_IDENT,
1730 pciebase + EIP197_XLX_USER_VECT_LUT2_ADDR);
1731 writel(EIP197_XLX_USER_VECT_LUT3_IDENT,
1732 pciebase + EIP197_XLX_USER_VECT_LUT3_ADDR);
1734 /* Enable all device interrupts */
1735 writel(GENMASK(31, 0),
1736 pciebase + EIP197_XLX_USER_INT_ENB_MSK);
1738 dev_err(dev, "Unrecognised IRQ block identifier %x\n",
1743 /* HW reset FPGA dev board */
1745 writel(1, priv->base + EIP197_XLX_GPIO_BASE);
1746 wmb(); /* maintain strict ordering for accesses here */
1747 /* deassert reset */
1748 writel(0, priv->base + EIP197_XLX_GPIO_BASE);
1749 wmb(); /* maintain strict ordering for accesses here */
1752 /* enable bus mastering */
1753 pci_set_master(pdev);
1755 /* Generic EIP97/EIP197 device probing */
1756 rc = safexcel_probe_generic(pdev, priv, 1);
1760 void safexcel_pci_remove(struct pci_dev *pdev)
1762 struct safexcel_crypto_priv *priv = pci_get_drvdata(pdev);
1765 safexcel_unregister_algorithms(priv);
1767 for (i = 0; i < priv->config.rings; i++)
1768 destroy_workqueue(priv->ring[i].workqueue);
1770 safexcel_hw_reset_rings(priv);
1773 static const struct pci_device_id safexcel_pci_ids[] = {
1775 PCI_DEVICE_SUB(PCI_VENDOR_ID_XILINX, 0x9038,
1777 .driver_data = EIP197_DEVBRD,
1782 MODULE_DEVICE_TABLE(pci, safexcel_pci_ids);
1784 static struct pci_driver safexcel_pci_driver = {
1785 .name = "crypto-safexcel",
1786 .id_table = safexcel_pci_ids,
1787 .probe = safexcel_pci_probe,
1788 .remove = safexcel_pci_remove,
1792 /* Unfortunately, we have to resort to global variables here */
1793 #if IS_ENABLED(CONFIG_PCI)
1794 int pcireg_rc = -EINVAL; /* Default safe value */
1796 #if IS_ENABLED(CONFIG_OF)
1797 int ofreg_rc = -EINVAL; /* Default safe value */
1800 static int __init safexcel_init(void)
1802 #if IS_ENABLED(CONFIG_PCI)
1803 /* Register PCI driver */
1804 pcireg_rc = pci_register_driver(&safexcel_pci_driver);
1807 #if IS_ENABLED(CONFIG_OF)
1808 /* Register platform driver */
1809 ofreg_rc = platform_driver_register(&crypto_safexcel);
1810 #if IS_ENABLED(CONFIG_PCI)
1811 /* Return success if either PCI or OF registered OK */
1812 return pcireg_rc ? ofreg_rc : 0;
1817 #if IS_ENABLED(CONFIG_PCI)
1825 static void __exit safexcel_exit(void)
1827 #if IS_ENABLED(CONFIG_OF)
1828 /* Unregister platform driver */
1830 platform_driver_unregister(&crypto_safexcel);
1833 #if IS_ENABLED(CONFIG_PCI)
1834 /* Unregister PCI driver if successfully registered before */
1836 pci_unregister_driver(&safexcel_pci_driver);
1840 module_init(safexcel_init);
1841 module_exit(safexcel_exit);
1843 MODULE_AUTHOR("Antoine Tenart <antoine.tenart@free-electrons.com>");
1844 MODULE_AUTHOR("Ofer Heifetz <oferh@marvell.com>");
1845 MODULE_AUTHOR("Igal Liberman <igall@marvell.com>");
1846 MODULE_DESCRIPTION("Support for SafeXcel cryptographic engines: EIP97 & EIP197");
1847 MODULE_LICENSE("GPL v2");