drivers/net/wireless/ath/ath10k/ce.c

   1 // SPDX-License-Identifier: ISC
   2 /*
   3  * Copyright (c) 2005-2011 Atheros Communications Inc.
   4  * Copyright (c) 2011-2017 Qualcomm Atheros, Inc.
   5  * Copyright (c) 2018 The Linux Foundation. All rights reserved.
   6  */
   7
   8 #include "hif.h"
   9 #include "ce.h"
  10 #include "debug.h"
  11
  12 /*
  13  * Support for Copy Engine hardware, which is mainly used for
  14  * communication between Host and Target over a PCIe interconnect.
  15  */
  16
  17 /*
  18  * A single CopyEngine (CE) comprises two "rings":
  19  *   a source ring
  20  *   a destination ring
  21  *
  22  * Each ring consists of a number of descriptors which specify
  23  * an address, length, and meta-data.
  24  *
  25  * Typically, one side of the PCIe/AHB/SNOC interconnect (Host or Target)
  26  * controls one ring and the other side controls the other ring.
  27  * The source side chooses when to initiate a transfer and it
  28  * chooses what to send (buffer address, length). The destination
  29  * side keeps a supply of "anonymous receive buffers" available and
  30  * it handles incoming data as it arrives (when the destination
  31  * receives an interrupt).
  32  *
  33  * The sender may send a simple buffer (address/length) or it may
  34  * send a small list of buffers.  When a small list is sent, hardware
  35  * "gathers" these and they end up in a single destination buffer
  36  * with a single interrupt.
  37  *
  38  * There are several "contexts" managed by this layer -- more, it
  39  * may seem -- than should be needed. These are provided mainly for
  40  * maximum flexibility and especially to facilitate a simpler HIF
  41  * implementation. There are per-CopyEngine recv, send, and watermark
  42  * contexts. These are supplied by the caller when a recv, send,
  43  * or watermark handler is established and they are echoed back to
  44  * the caller when the respective callbacks are invoked. There is
  45  * also a per-transfer context supplied by the caller when a buffer
  46  * (or sendlist) is sent and when a buffer is enqueued for recv.
  47  * These per-transfer contexts are echoed back to the caller when
  48  * the buffer is sent/received.
  49  */
  50
  51 static inline u32 shadow_sr_wr_ind_addr(struct ath10k *ar,
  52                                         struct ath10k_ce_pipe *ce_state)
  53 {
  54         u32 ce_id = ce_state->id;
  55         u32 addr = 0;
  56
  57         switch (ce_id) {
  58         case 0:
  59                 addr = 0x00032000;
  60                 break;
  61         case 3:
  62                 addr = 0x0003200C;
  63                 break;
  64         case 4:
  65                 addr = 0x00032010;
  66                 break;
  67         case 5:
  68                 addr = 0x00032014;
  69                 break;
  70         case 7:
  71                 addr = 0x0003201C;
  72                 break;
  73         default:
  74                 ath10k_warn(ar, "invalid CE id: %d", ce_id);
  75                 break;
  76         }
  77         return addr;
  78 }
  79
  80 static inline u32 shadow_dst_wr_ind_addr(struct ath10k *ar,
  81                                          struct ath10k_ce_pipe *ce_state)
  82 {
  83         u32 ce_id = ce_state->id;
  84         u32 addr = 0;
  85
  86         switch (ce_id) {
  87         case 1:
  88                 addr = 0x00032034;
  89                 break;
  90         case 2:
  91                 addr = 0x00032038;
  92                 break;
  93         case 5:
  94                 addr = 0x00032044;
  95                 break;
  96         case 7:
  97                 addr = 0x0003204C;
  98                 break;
  99         case 8:
 100                 addr = 0x00032050;
 101                 break;
 102         case 9:
 103                 addr = 0x00032054;
 104                 break;
 105         case 10:
 106                 addr = 0x00032058;
 107                 break;
 108         case 11:
 109                 addr = 0x0003205C;
 110                 break;
 111         default:
 112                 ath10k_warn(ar, "invalid CE id: %d", ce_id);
 113                 break;
 114         }
 115
 116         return addr;
 117 }
 118
 119 static inline unsigned int
 120 ath10k_set_ring_byte(unsigned int offset,
 121                      struct ath10k_hw_ce_regs_addr_map *addr_map)
 122 {
 123         return ((offset << addr_map->lsb) & addr_map->mask);
 124 }
 125
 126 static inline unsigned int
 127 ath10k_get_ring_byte(unsigned int offset,
 128                      struct ath10k_hw_ce_regs_addr_map *addr_map)
 129 {
 130         return ((offset & addr_map->mask) >> (addr_map->lsb));
 131 }
 132
 133 static inline u32 ath10k_ce_read32(struct ath10k *ar, u32 offset)
 134 {
 135         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 136
 137         return ce->bus_ops->read32(ar, offset);
 138 }
 139
 140 static inline void ath10k_ce_write32(struct ath10k *ar, u32 offset, u32 value)
 141 {
 142         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 143
 144         ce->bus_ops->write32(ar, offset, value);
 145 }
 146
 147 static inline void ath10k_ce_dest_ring_write_index_set(struct ath10k *ar,
 148                                                        u32 ce_ctrl_addr,
 149                                                        unsigned int n)
 150 {
 151         ath10k_ce_write32(ar, ce_ctrl_addr +
 152                           ar->hw_ce_regs->dst_wr_index_addr, n);
 153 }
 154
 155 static inline u32 ath10k_ce_dest_ring_write_index_get(struct ath10k *ar,
 156                                                       u32 ce_ctrl_addr)
 157 {
 158         return ath10k_ce_read32(ar, ce_ctrl_addr +
 159                                 ar->hw_ce_regs->dst_wr_index_addr);
 160 }
 161
 162 static inline void ath10k_ce_src_ring_write_index_set(struct ath10k *ar,
 163                                                       u32 ce_ctrl_addr,
 164                                                       unsigned int n)
 165 {
 166         ath10k_ce_write32(ar, ce_ctrl_addr +
 167                           ar->hw_ce_regs->sr_wr_index_addr, n);
 168 }
 169
 170 static inline u32 ath10k_ce_src_ring_write_index_get(struct ath10k *ar,
 171                                                      u32 ce_ctrl_addr)
 172 {
 173         return ath10k_ce_read32(ar, ce_ctrl_addr +
 174                                 ar->hw_ce_regs->sr_wr_index_addr);
 175 }
 176
 177 static inline u32 ath10k_ce_src_ring_read_index_from_ddr(struct ath10k *ar,
 178                                                          u32 ce_id)
 179 {
 180         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 181
 182         return ce->vaddr_rri[ce_id] & CE_DDR_RRI_MASK;
 183 }
 184
 185 static inline u32 ath10k_ce_src_ring_read_index_get(struct ath10k *ar,
 186                                                     u32 ce_ctrl_addr)
 187 {
 188         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 189         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 190         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 191         u32 index;
 192
 193         if (ar->hw_params.rri_on_ddr &&
 194             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 195                 index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_id);
 196         else
 197                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 198                                          ar->hw_ce_regs->current_srri_addr);
 199
 200         return index;
 201 }
 202
 203 static inline void
 204 ath10k_ce_shadow_src_ring_write_index_set(struct ath10k *ar,
 205                                           struct ath10k_ce_pipe *ce_state,
 206                                           unsigned int value)
 207 {
 208         ath10k_ce_write32(ar, shadow_sr_wr_ind_addr(ar, ce_state), value);
 209 }
 210
 211 static inline void
 212 ath10k_ce_shadow_dest_ring_write_index_set(struct ath10k *ar,
 213                                            struct ath10k_ce_pipe *ce_state,
 214                                            unsigned int value)
 215 {
 216         ath10k_ce_write32(ar, shadow_dst_wr_ind_addr(ar, ce_state), value);
 217 }
 218
 219 static inline void ath10k_ce_src_ring_base_addr_set(struct ath10k *ar,
 220                                                     u32 ce_id,
 221                                                     u64 addr)
 222 {
 223         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 224         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 225         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 226         u32 addr_lo = lower_32_bits(addr);
 227
 228         ath10k_ce_write32(ar, ce_ctrl_addr +
 229                           ar->hw_ce_regs->sr_base_addr_lo, addr_lo);
 230
 231         if (ce_state->ops->ce_set_src_ring_base_addr_hi) {
 232                 ce_state->ops->ce_set_src_ring_base_addr_hi(ar, ce_ctrl_addr,
 233                                                             addr);
 234         }
 235 }
 236
 237 static void ath10k_ce_set_src_ring_base_addr_hi(struct ath10k *ar,
 238                                                 u32 ce_ctrl_addr,
 239                                                 u64 addr)
 240 {
 241         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 242
 243         ath10k_ce_write32(ar, ce_ctrl_addr +
 244                           ar->hw_ce_regs->sr_base_addr_hi, addr_hi);
 245 }
 246
 247 static inline void ath10k_ce_src_ring_size_set(struct ath10k *ar,
 248                                                u32 ce_ctrl_addr,
 249                                                unsigned int n)
 250 {
 251         ath10k_ce_write32(ar, ce_ctrl_addr +
 252                           ar->hw_ce_regs->sr_size_addr, n);
 253 }
 254
 255 static inline void ath10k_ce_src_ring_dmax_set(struct ath10k *ar,
 256                                                u32 ce_ctrl_addr,
 257                                                unsigned int n)
 258 {
 259         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 260
 261         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 262                                           ctrl_regs->addr);
 263
 264         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 265                           (ctrl1_addr &  ~(ctrl_regs->dmax->mask)) |
 266                           ath10k_set_ring_byte(n, ctrl_regs->dmax));
 267 }
 268
 269 static inline void ath10k_ce_src_ring_byte_swap_set(struct ath10k *ar,
 270                                                     u32 ce_ctrl_addr,
 271                                                     unsigned int n)
 272 {
 273         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 274
 275         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 276                                           ctrl_regs->addr);
 277
 278         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 279                           (ctrl1_addr & ~(ctrl_regs->src_ring->mask)) |
 280                           ath10k_set_ring_byte(n, ctrl_regs->src_ring));
 281 }
 282
 283 static inline void ath10k_ce_dest_ring_byte_swap_set(struct ath10k *ar,
 284                                                      u32 ce_ctrl_addr,
 285                                                      unsigned int n)
 286 {
 287         struct ath10k_hw_ce_ctrl1 *ctrl_regs = ar->hw_ce_regs->ctrl1_regs;
 288
 289         u32 ctrl1_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 290                                           ctrl_regs->addr);
 291
 292         ath10k_ce_write32(ar, ce_ctrl_addr + ctrl_regs->addr,
 293                           (ctrl1_addr & ~(ctrl_regs->dst_ring->mask)) |
 294                           ath10k_set_ring_byte(n, ctrl_regs->dst_ring));
 295 }
 296
 297 static inline
 298         u32 ath10k_ce_dest_ring_read_index_from_ddr(struct ath10k *ar, u32 ce_id)
 299 {
 300         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 301
 302         return (ce->vaddr_rri[ce_id] >> CE_DDR_DRRI_SHIFT) &
 303                 CE_DDR_RRI_MASK;
 304 }
 305
 306 static inline u32 ath10k_ce_dest_ring_read_index_get(struct ath10k *ar,
 307                                                      u32 ce_ctrl_addr)
 308 {
 309         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 310         u32 ce_id = COPY_ENGINE_ID(ce_ctrl_addr);
 311         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 312         u32 index;
 313
 314         if (ar->hw_params.rri_on_ddr &&
 315             (ce_state->attr_flags & CE_ATTR_DIS_INTR))
 316                 index = ath10k_ce_dest_ring_read_index_from_ddr(ar, ce_id);
 317         else
 318                 index = ath10k_ce_read32(ar, ce_ctrl_addr +
 319                                          ar->hw_ce_regs->current_drri_addr);
 320
 321         return index;
 322 }
 323
 324 static inline void ath10k_ce_dest_ring_base_addr_set(struct ath10k *ar,
 325                                                      u32 ce_id,
 326                                                      u64 addr)
 327 {
 328         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 329         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
 330         u32 ce_ctrl_addr = ath10k_ce_base_address(ar, ce_id);
 331         u32 addr_lo = lower_32_bits(addr);
 332
 333         ath10k_ce_write32(ar, ce_ctrl_addr +
 334                           ar->hw_ce_regs->dr_base_addr_lo, addr_lo);
 335
 336         if (ce_state->ops->ce_set_dest_ring_base_addr_hi) {
 337                 ce_state->ops->ce_set_dest_ring_base_addr_hi(ar, ce_ctrl_addr,
 338                                                              addr);
 339         }
 340 }
 341
 342 static void ath10k_ce_set_dest_ring_base_addr_hi(struct ath10k *ar,
 343                                                  u32 ce_ctrl_addr,
 344                                                  u64 addr)
 345 {
 346         u32 addr_hi = upper_32_bits(addr) & CE_DESC_ADDR_HI_MASK;
 347         u32 reg_value;
 348
 349         reg_value = ath10k_ce_read32(ar, ce_ctrl_addr +
 350                                      ar->hw_ce_regs->dr_base_addr_hi);
 351         reg_value &= ~CE_DESC_ADDR_HI_MASK;
 352         reg_value |= addr_hi;
 353         ath10k_ce_write32(ar, ce_ctrl_addr +
 354                           ar->hw_ce_regs->dr_base_addr_hi, reg_value);
 355 }
 356
 357 static inline void ath10k_ce_dest_ring_size_set(struct ath10k *ar,
 358                                                 u32 ce_ctrl_addr,
 359                                                 unsigned int n)
 360 {
 361         ath10k_ce_write32(ar, ce_ctrl_addr +
 362                           ar->hw_ce_regs->dr_size_addr, n);
 363 }
 364
 365 static inline void ath10k_ce_src_ring_highmark_set(struct ath10k *ar,
 366                                                    u32 ce_ctrl_addr,
 367                                                    unsigned int n)
 368 {
 369         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 370         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 371
 372         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 373                           (addr & ~(srcr_wm->wm_high->mask)) |
 374                           (ath10k_set_ring_byte(n, srcr_wm->wm_high)));
 375 }
 376
 377 static inline void ath10k_ce_src_ring_lowmark_set(struct ath10k *ar,
 378                                                   u32 ce_ctrl_addr,
 379                                                   unsigned int n)
 380 {
 381         struct ath10k_hw_ce_dst_src_wm_regs *srcr_wm = ar->hw_ce_regs->wm_srcr;
 382         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + srcr_wm->addr);
 383
 384         ath10k_ce_write32(ar, ce_ctrl_addr + srcr_wm->addr,
 385                           (addr & ~(srcr_wm->wm_low->mask)) |
 386                           (ath10k_set_ring_byte(n, srcr_wm->wm_low)));
 387 }
 388
 389 static inline void ath10k_ce_dest_ring_highmark_set(struct ath10k *ar,
 390                                                     u32 ce_ctrl_addr,
 391                                                     unsigned int n)
 392 {
 393         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 394         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 395
 396         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 397                           (addr & ~(dstr_wm->wm_high->mask)) |
 398                           (ath10k_set_ring_byte(n, dstr_wm->wm_high)));
 399 }
 400
 401 static inline void ath10k_ce_dest_ring_lowmark_set(struct ath10k *ar,
 402                                                    u32 ce_ctrl_addr,
 403                                                    unsigned int n)
 404 {
 405         struct ath10k_hw_ce_dst_src_wm_regs *dstr_wm = ar->hw_ce_regs->wm_dstr;
 406         u32 addr = ath10k_ce_read32(ar, ce_ctrl_addr + dstr_wm->addr);
 407
 408         ath10k_ce_write32(ar, ce_ctrl_addr + dstr_wm->addr,
 409                           (addr & ~(dstr_wm->wm_low->mask)) |
 410                           (ath10k_set_ring_byte(n, dstr_wm->wm_low)));
 411 }
 412
 413 static inline void ath10k_ce_copy_complete_inter_enable(struct ath10k *ar,
 414                                                         u32 ce_ctrl_addr)
 415 {
 416         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 417
 418         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 419                                             ar->hw_ce_regs->host_ie_addr);
 420
 421         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 422                           host_ie_addr | host_ie->copy_complete->mask);
 423 }
 424
 425 static inline void ath10k_ce_copy_complete_intr_disable(struct ath10k *ar,
 426                                                         u32 ce_ctrl_addr)
 427 {
 428         struct ath10k_hw_ce_host_ie *host_ie = ar->hw_ce_regs->host_ie;
 429
 430         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 431                                             ar->hw_ce_regs->host_ie_addr);
 432
 433         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 434                           host_ie_addr & ~(host_ie->copy_complete->mask));
 435 }
 436
 437 static inline void ath10k_ce_watermark_intr_disable(struct ath10k *ar,
 438                                                     u32 ce_ctrl_addr)
 439 {
 440         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 441
 442         u32 host_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 443                                             ar->hw_ce_regs->host_ie_addr);
 444
 445         ath10k_ce_write32(ar, ce_ctrl_addr + ar->hw_ce_regs->host_ie_addr,
 446                           host_ie_addr & ~(wm_regs->wm_mask));
 447 }
 448
 449 static inline void ath10k_ce_error_intr_enable(struct ath10k *ar,
 450                                                u32 ce_ctrl_addr)
 451 {
 452         struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 453
 454         u32 misc_ie_addr = ath10k_ce_read32(ar, ce_ctrl_addr +
 455                                             ar->hw_ce_regs->misc_ie_addr);
 456
 457         ath10k_ce_write32(ar,
 458                           ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 459                           misc_ie_addr | misc_regs->err_mask);
 460 }
 461
 462 static inline void ath10k_ce_error_intr_disable(struct ath10k *ar,
 463                                                 u32 ce_ctrl_addr)
 464 {
 465         struct ath10k_hw_ce_misc_regs *misc_regs = ar->hw_ce_regs->misc_regs;
 466
 467         u32 misc_ie_addr = ath10k_ce_read32(ar,
 468                         ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr);
 469
 470         ath10k_ce_write32(ar,
 471                           ce_ctrl_addr + ar->hw_ce_regs->misc_ie_addr,
 472                           misc_ie_addr & ~(misc_regs->err_mask));
 473 }
 474
 475 static inline void ath10k_ce_engine_int_status_clear(struct ath10k *ar,
 476                                                      u32 ce_ctrl_addr,
 477                                                      unsigned int mask)
 478 {
 479         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
 480
 481         ath10k_ce_write32(ar, ce_ctrl_addr + wm_regs->addr, mask);
 482 }
 483
 484 /*
 485  * Guts of ath10k_ce_send.
 486  * The caller takes responsibility for any needed locking.
 487  */
 488 static int _ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 489                                   void *per_transfer_context,
 490                                   dma_addr_t buffer,
 491                                   unsigned int nbytes,
 492                                   unsigned int transfer_id,
 493                                   unsigned int flags)
 494 {
 495         struct ath10k *ar = ce_state->ar;
 496         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 497         struct ce_desc *desc, sdesc;
 498         unsigned int nentries_mask = src_ring->nentries_mask;
 499         unsigned int sw_index = src_ring->sw_index;
 500         unsigned int write_index = src_ring->write_index;
 501         u32 ctrl_addr = ce_state->ctrl_addr;
 502         u32 desc_flags = 0;
 503         int ret = 0;
 504
 505         if (nbytes > ce_state->src_sz_max)
 506                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 507                             __func__, nbytes, ce_state->src_sz_max);
 508
 509         if (unlikely(CE_RING_DELTA(nentries_mask,
 510                                    write_index, sw_index - 1) <= 0)) {
 511                 ret = -ENOSR;
 512                 goto exit;
 513         }
 514
 515         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
 516                                    write_index);
 517
 518         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 519
 520         if (flags & CE_SEND_FLAG_GATHER)
 521                 desc_flags |= CE_DESC_FLAGS_GATHER;
 522         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 523                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 524
 525         sdesc.addr   = __cpu_to_le32(buffer);
 526         sdesc.nbytes = __cpu_to_le16(nbytes);
 527         sdesc.flags  = __cpu_to_le16(desc_flags);
 528
 529         *desc = sdesc;
 530
 531         src_ring->per_transfer_context[write_index] = per_transfer_context;
 532
 533         /* Update Source Ring Write Index */
 534         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 535
 536         /* WORKAROUND */
 537         if (!(flags & CE_SEND_FLAG_GATHER))
 538                 ath10k_ce_src_ring_write_index_set(ar, ctrl_addr, write_index);
 539
 540         src_ring->write_index = write_index;
 541 exit:
 542         return ret;
 543 }
 544
 545 static int _ath10k_ce_send_nolock_64(struct ath10k_ce_pipe *ce_state,
 546                                      void *per_transfer_context,
 547                                      dma_addr_t buffer,
 548                                      unsigned int nbytes,
 549                                      unsigned int transfer_id,
 550                                      unsigned int flags)
 551 {
 552         struct ath10k *ar = ce_state->ar;
 553         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
 554         struct ce_desc_64 *desc, sdesc;
 555         unsigned int nentries_mask = src_ring->nentries_mask;
 556         unsigned int sw_index;
 557         unsigned int write_index = src_ring->write_index;
 558         u32 ctrl_addr = ce_state->ctrl_addr;
 559         __le32 *addr;
 560         u32 desc_flags = 0;
 561         int ret = 0;
 562
 563         if (test_bit(ATH10K_FLAG_CRASH_FLUSH, &ar->dev_flags))
 564                 return -ESHUTDOWN;
 565
 566         if (nbytes > ce_state->src_sz_max)
 567                 ath10k_warn(ar, "%s: send more we can (nbytes: %d, max: %d)\n",
 568                             __func__, nbytes, ce_state->src_sz_max);
 569
 570         if (ar->hw_params.rri_on_ddr)
 571                 sw_index = ath10k_ce_src_ring_read_index_from_ddr(ar, ce_state->id);
 572         else
 573                 sw_index = src_ring->sw_index;
 574
 575         if (unlikely(CE_RING_DELTA(nentries_mask,
 576                                    write_index, sw_index - 1) <= 0)) {
 577                 ret = -ENOSR;
 578                 goto exit;
 579         }
 580
 581         desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
 582                                       write_index);
 583
 584         desc_flags |= SM(transfer_id, CE_DESC_FLAGS_META_DATA);
 585
 586         if (flags & CE_SEND_FLAG_GATHER)
 587                 desc_flags |= CE_DESC_FLAGS_GATHER;
 588
 589         if (flags & CE_SEND_FLAG_BYTE_SWAP)
 590                 desc_flags |= CE_DESC_FLAGS_BYTE_SWAP;
 591
 592         addr = (__le32 *)&sdesc.addr;
 593
 594         flags |= upper_32_bits(buffer) & CE_DESC_ADDR_HI_MASK;
 595         addr[0] = __cpu_to_le32(buffer);
 596         addr[1] = __cpu_to_le32(flags);
 597         if (flags & CE_SEND_FLAG_GATHER)
 598                 addr[1] |= __cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER);
 599         else
 600                 addr[1] &= ~(__cpu_to_le32(CE_WCN3990_DESC_FLAGS_GATHER));
 601
 602         sdesc.nbytes = __cpu_to_le16(nbytes);
 603         sdesc.flags  = __cpu_to_le16(desc_flags);
 604
 605         *desc = sdesc;
 606
 607         src_ring->per_transfer_context[write_index] = per_transfer_context;
 608
 609         /* Update Source Ring Write Index */
 610         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 611
 612         if (!(flags & CE_SEND_FLAG_GATHER)) {
 613                 if (ar->hw_params.shadow_reg_support)
 614                         ath10k_ce_shadow_src_ring_write_index_set(ar, ce_state,
 615                                                                   write_index);
 616                 else
 617                         ath10k_ce_src_ring_write_index_set(ar, ctrl_addr,
 618                                                            write_index);
 619         }
 620
 621         src_ring->write_index = write_index;
 622 exit:
 623         return ret;
 624 }
 625
 626 int ath10k_ce_send_nolock(struct ath10k_ce_pipe *ce_state,
 627                           void *per_transfer_context,
 628                           dma_addr_t buffer,
 629                           unsigned int nbytes,
 630                           unsigned int transfer_id,
 631                           unsigned int flags)
 632 {
 633         return ce_state->ops->ce_send_nolock(ce_state, per_transfer_context,
 634                                     buffer, nbytes, transfer_id, flags);
 635 }
 636 EXPORT_SYMBOL(ath10k_ce_send_nolock);
 637
 638 void __ath10k_ce_send_revert(struct ath10k_ce_pipe *pipe)
 639 {
 640         struct ath10k *ar = pipe->ar;
 641         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 642         struct ath10k_ce_ring *src_ring = pipe->src_ring;
 643         u32 ctrl_addr = pipe->ctrl_addr;
 644
 645         lockdep_assert_held(&ce->ce_lock);
 646
 647         /*
 648          * This function must be called only if there is an incomplete
 649          * scatter-gather transfer (before index register is updated)
 650          * that needs to be cleaned up.
 651          */
 652         if (WARN_ON_ONCE(src_ring->write_index == src_ring->sw_index))
 653                 return;
 654
 655         if (WARN_ON_ONCE(src_ring->write_index ==
 656                          ath10k_ce_src_ring_write_index_get(ar, ctrl_addr)))
 657                 return;
 658
 659         src_ring->write_index--;
 660         src_ring->write_index &= src_ring->nentries_mask;
 661
 662         src_ring->per_transfer_context[src_ring->write_index] = NULL;
 663 }
 664 EXPORT_SYMBOL(__ath10k_ce_send_revert);
 665
 666 int ath10k_ce_send(struct ath10k_ce_pipe *ce_state,
 667                    void *per_transfer_context,
 668                    dma_addr_t buffer,
 669                    unsigned int nbytes,
 670                    unsigned int transfer_id,
 671                    unsigned int flags)
 672 {
 673         struct ath10k *ar = ce_state->ar;
 674         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 675         int ret;
 676
 677         spin_lock_bh(&ce->ce_lock);
 678         ret = ath10k_ce_send_nolock(ce_state, per_transfer_context,
 679                                     buffer, nbytes, transfer_id, flags);
 680         spin_unlock_bh(&ce->ce_lock);
 681
 682         return ret;
 683 }
 684 EXPORT_SYMBOL(ath10k_ce_send);
 685
 686 int ath10k_ce_num_free_src_entries(struct ath10k_ce_pipe *pipe)
 687 {
 688         struct ath10k *ar = pipe->ar;
 689         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 690         int delta;
 691
 692         spin_lock_bh(&ce->ce_lock);
 693         delta = CE_RING_DELTA(pipe->src_ring->nentries_mask,
 694                               pipe->src_ring->write_index,
 695                               pipe->src_ring->sw_index - 1);
 696         spin_unlock_bh(&ce->ce_lock);
 697
 698         return delta;
 699 }
 700 EXPORT_SYMBOL(ath10k_ce_num_free_src_entries);
 701
 702 int __ath10k_ce_rx_num_free_bufs(struct ath10k_ce_pipe *pipe)
 703 {
 704         struct ath10k *ar = pipe->ar;
 705         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 706         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 707         unsigned int nentries_mask = dest_ring->nentries_mask;
 708         unsigned int write_index = dest_ring->write_index;
 709         unsigned int sw_index = dest_ring->sw_index;
 710
 711         lockdep_assert_held(&ce->ce_lock);
 712
 713         return CE_RING_DELTA(nentries_mask, write_index, sw_index - 1);
 714 }
 715 EXPORT_SYMBOL(__ath10k_ce_rx_num_free_bufs);
 716
 717 static int __ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 718                                    dma_addr_t paddr)
 719 {
 720         struct ath10k *ar = pipe->ar;
 721         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 722         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 723         unsigned int nentries_mask = dest_ring->nentries_mask;
 724         unsigned int write_index = dest_ring->write_index;
 725         unsigned int sw_index = dest_ring->sw_index;
 726         struct ce_desc *base = dest_ring->base_addr_owner_space;
 727         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, write_index);
 728         u32 ctrl_addr = pipe->ctrl_addr;
 729
 730         lockdep_assert_held(&ce->ce_lock);
 731
 732         if ((pipe->id != 5) &&
 733             CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 734                 return -ENOSPC;
 735
 736         desc->addr = __cpu_to_le32(paddr);
 737         desc->nbytes = 0;
 738
 739         dest_ring->per_transfer_context[write_index] = ctx;
 740         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 741         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 742         dest_ring->write_index = write_index;
 743
 744         return 0;
 745 }
 746
 747 static int __ath10k_ce_rx_post_buf_64(struct ath10k_ce_pipe *pipe,
 748                                       void *ctx,
 749                                       dma_addr_t paddr)
 750 {
 751         struct ath10k *ar = pipe->ar;
 752         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 753         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 754         unsigned int nentries_mask = dest_ring->nentries_mask;
 755         unsigned int write_index = dest_ring->write_index;
 756         unsigned int sw_index = dest_ring->sw_index;
 757         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 758         struct ce_desc_64 *desc =
 759                         CE_DEST_RING_TO_DESC_64(base, write_index);
 760         u32 ctrl_addr = pipe->ctrl_addr;
 761
 762         lockdep_assert_held(&ce->ce_lock);
 763
 764         if (CE_RING_DELTA(nentries_mask, write_index, sw_index - 1) == 0)
 765                 return -ENOSPC;
 766
 767         desc->addr = __cpu_to_le64(paddr);
 768         desc->addr &= __cpu_to_le64(CE_DESC_ADDR_MASK);
 769
 770         desc->nbytes = 0;
 771
 772         dest_ring->per_transfer_context[write_index] = ctx;
 773         write_index = CE_RING_IDX_INCR(nentries_mask, write_index);
 774         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 775         dest_ring->write_index = write_index;
 776
 777         return 0;
 778 }
 779
 780 void ath10k_ce_rx_update_write_idx(struct ath10k_ce_pipe *pipe, u32 nentries)
 781 {
 782         struct ath10k *ar = pipe->ar;
 783         struct ath10k_ce_ring *dest_ring = pipe->dest_ring;
 784         unsigned int nentries_mask = dest_ring->nentries_mask;
 785         unsigned int write_index = dest_ring->write_index;
 786         u32 ctrl_addr = pipe->ctrl_addr;
 787         u32 cur_write_idx = ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
 788
 789         /* Prevent CE ring stuck issue that will occur when ring is full.
 790          * Make sure that write index is 1 less than read index.
 791          */
 792         if (((cur_write_idx + nentries) & nentries_mask) == dest_ring->sw_index)
 793                 nentries -= 1;
 794
 795         write_index = CE_RING_IDX_ADD(nentries_mask, write_index, nentries);
 796         ath10k_ce_dest_ring_write_index_set(ar, ctrl_addr, write_index);
 797         dest_ring->write_index = write_index;
 798 }
 799 EXPORT_SYMBOL(ath10k_ce_rx_update_write_idx);
 800
 801 int ath10k_ce_rx_post_buf(struct ath10k_ce_pipe *pipe, void *ctx,
 802                           dma_addr_t paddr)
 803 {
 804         struct ath10k *ar = pipe->ar;
 805         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 806         int ret;
 807
 808         spin_lock_bh(&ce->ce_lock);
 809         ret = pipe->ops->ce_rx_post_buf(pipe, ctx, paddr);
 810         spin_unlock_bh(&ce->ce_lock);
 811
 812         return ret;
 813 }
 814 EXPORT_SYMBOL(ath10k_ce_rx_post_buf);
 815
 816 /*
 817  * Guts of ath10k_ce_completed_recv_next.
 818  * The caller takes responsibility for any necessary locking.
 819  */
 820 static int
 821          _ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 822                                                void **per_transfer_contextp,
 823                                                unsigned int *nbytesp)
 824 {
 825         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 826         unsigned int nentries_mask = dest_ring->nentries_mask;
 827         unsigned int sw_index = dest_ring->sw_index;
 828
 829         struct ce_desc *base = dest_ring->base_addr_owner_space;
 830         struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 831         struct ce_desc sdesc;
 832         u16 nbytes;
 833
 834         /* Copy in one go for performance reasons */
 835         sdesc = *desc;
 836
 837         nbytes = __le16_to_cpu(sdesc.nbytes);
 838         if (nbytes == 0) {
 839                 /*
 840                  * This closes a relatively unusual race where the Host
 841                  * sees the updated DRRI before the update to the
 842                  * corresponding descriptor has completed. We treat this
 843                  * as a descriptor that is not yet done.
 844                  */
 845                 return -EIO;
 846         }
 847
 848         desc->nbytes = 0;
 849
 850         /* Return data from completed destination descriptor */
 851         *nbytesp = nbytes;
 852
 853         if (per_transfer_contextp)
 854                 *per_transfer_contextp =
 855                         dest_ring->per_transfer_context[sw_index];
 856
 857         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 858          * So update transfer context all CEs except CE5.
 859          */
 860         if (ce_state->id != 5)
 861                 dest_ring->per_transfer_context[sw_index] = NULL;
 862
 863         /* Update sw_index */
 864         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 865         dest_ring->sw_index = sw_index;
 866
 867         return 0;
 868 }
 869
 870 static int
 871 _ath10k_ce_completed_recv_next_nolock_64(struct ath10k_ce_pipe *ce_state,
 872                                          void **per_transfer_contextp,
 873                                          unsigned int *nbytesp)
 874 {
 875         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
 876         unsigned int nentries_mask = dest_ring->nentries_mask;
 877         unsigned int sw_index = dest_ring->sw_index;
 878         struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
 879         struct ce_desc_64 *desc =
 880                 CE_DEST_RING_TO_DESC_64(base, sw_index);
 881         struct ce_desc_64 sdesc;
 882         u16 nbytes;
 883
 884         /* Copy in one go for performance reasons */
 885         sdesc = *desc;
 886
 887         nbytes = __le16_to_cpu(sdesc.nbytes);
 888         if (nbytes == 0) {
 889                 /* This closes a relatively unusual race where the Host
 890                  * sees the updated DRRI before the update to the
 891                  * corresponding descriptor has completed. We treat this
 892                  * as a descriptor that is not yet done.
 893                  */
 894                 return -EIO;
 895         }
 896
 897         desc->nbytes = 0;
 898
 899         /* Return data from completed destination descriptor */
 900         *nbytesp = nbytes;
 901
 902         if (per_transfer_contextp)
 903                 *per_transfer_contextp =
 904                         dest_ring->per_transfer_context[sw_index];
 905
 906         /* Copy engine 5 (HTT Rx) will reuse the same transfer context.
 907          * So update transfer context all CEs except CE5.
 908          */
 909         if (ce_state->id != 5)
 910                 dest_ring->per_transfer_context[sw_index] = NULL;
 911
 912         /* Update sw_index */
 913         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 914         dest_ring->sw_index = sw_index;
 915
 916         return 0;
 917 }
 918
 919 int ath10k_ce_completed_recv_next_nolock(struct ath10k_ce_pipe *ce_state,
 920                                          void **per_transfer_ctx,
 921                                          unsigned int *nbytesp)
 922 {
 923         return ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 924                                                             per_transfer_ctx,
 925                                                             nbytesp);
 926 }
 927 EXPORT_SYMBOL(ath10k_ce_completed_recv_next_nolock);
 928
 929 int ath10k_ce_completed_recv_next(struct ath10k_ce_pipe *ce_state,
 930                                   void **per_transfer_contextp,
 931                                   unsigned int *nbytesp)
 932 {
 933         struct ath10k *ar = ce_state->ar;
 934         struct ath10k_ce *ce = ath10k_ce_priv(ar);
 935         int ret;
 936
 937         spin_lock_bh(&ce->ce_lock);
 938         ret = ce_state->ops->ce_completed_recv_next_nolock(ce_state,
 939                                                    per_transfer_contextp,
 940                                                    nbytesp);
 941
 942         spin_unlock_bh(&ce->ce_lock);
 943
 944         return ret;
 945 }
 946 EXPORT_SYMBOL(ath10k_ce_completed_recv_next);
 947
 948 static int _ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
 949                                        void **per_transfer_contextp,
 950                                        dma_addr_t *bufferp)
 951 {
 952         struct ath10k_ce_ring *dest_ring;
 953         unsigned int nentries_mask;
 954         unsigned int sw_index;
 955         unsigned int write_index;
 956         int ret;
 957         struct ath10k *ar;
 958         struct ath10k_ce *ce;
 959
 960         dest_ring = ce_state->dest_ring;
 961
 962         if (!dest_ring)
 963                 return -EIO;
 964
 965         ar = ce_state->ar;
 966         ce = ath10k_ce_priv(ar);
 967
 968         spin_lock_bh(&ce->ce_lock);
 969
 970         nentries_mask = dest_ring->nentries_mask;
 971         sw_index = dest_ring->sw_index;
 972         write_index = dest_ring->write_index;
 973         if (write_index != sw_index) {
 974                 struct ce_desc *base = dest_ring->base_addr_owner_space;
 975                 struct ce_desc *desc = CE_DEST_RING_TO_DESC(base, sw_index);
 976
 977                 /* Return data from completed destination descriptor */
 978                 *bufferp = __le32_to_cpu(desc->addr);
 979
 980                 if (per_transfer_contextp)
 981                         *per_transfer_contextp =
 982                                 dest_ring->per_transfer_context[sw_index];
 983
 984                 /* sanity */
 985                 dest_ring->per_transfer_context[sw_index] = NULL;
 986                 desc->nbytes = 0;
 987
 988                 /* Update sw_index */
 989                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
 990                 dest_ring->sw_index = sw_index;
 991                 ret = 0;
 992         } else {
 993                 ret = -EIO;
 994         }
 995
 996         spin_unlock_bh(&ce->ce_lock);
 997
 998         return ret;
 999 }
1000
1001 static int _ath10k_ce_revoke_recv_next_64(struct ath10k_ce_pipe *ce_state,
1002                                           void **per_transfer_contextp,
1003                                           dma_addr_t *bufferp)
1004 {
1005         struct ath10k_ce_ring *dest_ring;
1006         unsigned int nentries_mask;
1007         unsigned int sw_index;
1008         unsigned int write_index;
1009         int ret;
1010         struct ath10k *ar;
1011         struct ath10k_ce *ce;
1012
1013         dest_ring = ce_state->dest_ring;
1014
1015         if (!dest_ring)
1016                 return -EIO;
1017
1018         ar = ce_state->ar;
1019         ce = ath10k_ce_priv(ar);
1020
1021         spin_lock_bh(&ce->ce_lock);
1022
1023         nentries_mask = dest_ring->nentries_mask;
1024         sw_index = dest_ring->sw_index;
1025         write_index = dest_ring->write_index;
1026         if (write_index != sw_index) {
1027                 struct ce_desc_64 *base = dest_ring->base_addr_owner_space;
1028                 struct ce_desc_64 *desc =
1029                         CE_DEST_RING_TO_DESC_64(base, sw_index);
1030
1031                 /* Return data from completed destination descriptor */
1032                 *bufferp = __le64_to_cpu(desc->addr);
1033
1034                 if (per_transfer_contextp)
1035                         *per_transfer_contextp =
1036                                 dest_ring->per_transfer_context[sw_index];
1037
1038                 /* sanity */
1039                 dest_ring->per_transfer_context[sw_index] = NULL;
1040                 desc->nbytes = 0;
1041
1042                 /* Update sw_index */
1043                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1044                 dest_ring->sw_index = sw_index;
1045                 ret = 0;
1046         } else {
1047                 ret = -EIO;
1048         }
1049
1050         spin_unlock_bh(&ce->ce_lock);
1051
1052         return ret;
1053 }
1054
1055 int ath10k_ce_revoke_recv_next(struct ath10k_ce_pipe *ce_state,
1056                                void **per_transfer_contextp,
1057                                dma_addr_t *bufferp)
1058 {
1059         return ce_state->ops->ce_revoke_recv_next(ce_state,
1060                                                   per_transfer_contextp,
1061                                                   bufferp);
1062 }
1063 EXPORT_SYMBOL(ath10k_ce_revoke_recv_next);
1064
1065 /*
1066  * Guts of ath10k_ce_completed_send_next.
1067  * The caller takes responsibility for any necessary locking.
1068  */
1069 static int _ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1070                                                  void **per_transfer_contextp)
1071 {
1072         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1073         u32 ctrl_addr = ce_state->ctrl_addr;
1074         struct ath10k *ar = ce_state->ar;
1075         unsigned int nentries_mask = src_ring->nentries_mask;
1076         unsigned int sw_index = src_ring->sw_index;
1077         unsigned int read_index;
1078         struct ce_desc *desc;
1079
1080         if (src_ring->hw_index == sw_index) {
1081                 /*
1082                  * The SW completion index has caught up with the cached
1083                  * version of the HW completion index.
1084                  * Update the cached HW completion index to see whether
1085                  * the SW has really caught up to the HW, or if the cached
1086                  * value of the HW index has become stale.
1087                  */
1088
1089                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1090                 if (read_index == 0xffffffff)
1091                         return -ENODEV;
1092
1093                 read_index &= nentries_mask;
1094                 src_ring->hw_index = read_index;
1095         }
1096
1097         if (ar->hw_params.rri_on_ddr)
1098                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1099         else
1100                 read_index = src_ring->hw_index;
1101
1102         if (read_index == sw_index)
1103                 return -EIO;
1104
1105         if (per_transfer_contextp)
1106                 *per_transfer_contextp =
1107                         src_ring->per_transfer_context[sw_index];
1108
1109         /* sanity */
1110         src_ring->per_transfer_context[sw_index] = NULL;
1111         desc = CE_SRC_RING_TO_DESC(src_ring->base_addr_owner_space,
1112                                    sw_index);
1113         desc->nbytes = 0;
1114
1115         /* Update sw_index */
1116         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1117         src_ring->sw_index = sw_index;
1118
1119         return 0;
1120 }
1121
1122 static int _ath10k_ce_completed_send_next_nolock_64(struct ath10k_ce_pipe *ce_state,
1123                                                     void **per_transfer_contextp)
1124 {
1125         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1126         u32 ctrl_addr = ce_state->ctrl_addr;
1127         struct ath10k *ar = ce_state->ar;
1128         unsigned int nentries_mask = src_ring->nentries_mask;
1129         unsigned int sw_index = src_ring->sw_index;
1130         unsigned int read_index;
1131         struct ce_desc_64 *desc;
1132
1133         if (src_ring->hw_index == sw_index) {
1134                 /*
1135                  * The SW completion index has caught up with the cached
1136                  * version of the HW completion index.
1137                  * Update the cached HW completion index to see whether
1138                  * the SW has really caught up to the HW, or if the cached
1139                  * value of the HW index has become stale.
1140                  */
1141
1142                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1143                 if (read_index == 0xffffffff)
1144                         return -ENODEV;
1145
1146                 read_index &= nentries_mask;
1147                 src_ring->hw_index = read_index;
1148         }
1149
1150         if (ar->hw_params.rri_on_ddr)
1151                 read_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1152         else
1153                 read_index = src_ring->hw_index;
1154
1155         if (read_index == sw_index)
1156                 return -EIO;
1157
1158         if (per_transfer_contextp)
1159                 *per_transfer_contextp =
1160                         src_ring->per_transfer_context[sw_index];
1161
1162         /* sanity */
1163         src_ring->per_transfer_context[sw_index] = NULL;
1164         desc = CE_SRC_RING_TO_DESC_64(src_ring->base_addr_owner_space,
1165                                       sw_index);
1166         desc->nbytes = 0;
1167
1168         /* Update sw_index */
1169         sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1170         src_ring->sw_index = sw_index;
1171
1172         return 0;
1173 }
1174
1175 int ath10k_ce_completed_send_next_nolock(struct ath10k_ce_pipe *ce_state,
1176                                          void **per_transfer_contextp)
1177 {
1178         return ce_state->ops->ce_completed_send_next_nolock(ce_state,
1179                                                             per_transfer_contextp);
1180 }
1181 EXPORT_SYMBOL(ath10k_ce_completed_send_next_nolock);
1182
1183 static void ath10k_ce_extract_desc_data(struct ath10k *ar,
1184                                         struct ath10k_ce_ring *src_ring,
1185                                         u32 sw_index,
1186                                         dma_addr_t *bufferp,
1187                                         u32 *nbytesp,
1188                                         u32 *transfer_idp)
1189 {
1190                 struct ce_desc *base = src_ring->base_addr_owner_space;
1191                 struct ce_desc *desc = CE_SRC_RING_TO_DESC(base, sw_index);
1192
1193                 /* Return data from completed source descriptor */
1194                 *bufferp = __le32_to_cpu(desc->addr);
1195                 *nbytesp = __le16_to_cpu(desc->nbytes);
1196                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1197                                    CE_DESC_FLAGS_META_DATA);
1198 }
1199
1200 static void ath10k_ce_extract_desc_data_64(struct ath10k *ar,
1201                                            struct ath10k_ce_ring *src_ring,
1202                                            u32 sw_index,
1203                                            dma_addr_t *bufferp,
1204                                            u32 *nbytesp,
1205                                            u32 *transfer_idp)
1206 {
1207                 struct ce_desc_64 *base = src_ring->base_addr_owner_space;
1208                 struct ce_desc_64 *desc =
1209                         CE_SRC_RING_TO_DESC_64(base, sw_index);
1210
1211                 /* Return data from completed source descriptor */
1212                 *bufferp = __le64_to_cpu(desc->addr);
1213                 *nbytesp = __le16_to_cpu(desc->nbytes);
1214                 *transfer_idp = MS(__le16_to_cpu(desc->flags),
1215                                    CE_DESC_FLAGS_META_DATA);
1216 }
1217
1218 /* NB: Modeled after ath10k_ce_completed_send_next */
1219 int ath10k_ce_cancel_send_next(struct ath10k_ce_pipe *ce_state,
1220                                void **per_transfer_contextp,
1221                                dma_addr_t *bufferp,
1222                                unsigned int *nbytesp,
1223                                unsigned int *transfer_idp)
1224 {
1225         struct ath10k_ce_ring *src_ring;
1226         unsigned int nentries_mask;
1227         unsigned int sw_index;
1228         unsigned int write_index;
1229         int ret;
1230         struct ath10k *ar;
1231         struct ath10k_ce *ce;
1232
1233         src_ring = ce_state->src_ring;
1234
1235         if (!src_ring)
1236                 return -EIO;
1237
1238         ar = ce_state->ar;
1239         ce = ath10k_ce_priv(ar);
1240
1241         spin_lock_bh(&ce->ce_lock);
1242
1243         nentries_mask = src_ring->nentries_mask;
1244         sw_index = src_ring->sw_index;
1245         write_index = src_ring->write_index;
1246
1247         if (write_index != sw_index) {
1248                 ce_state->ops->ce_extract_desc_data(ar, src_ring, sw_index,
1249                                                     bufferp, nbytesp,
1250                                                     transfer_idp);
1251
1252                 if (per_transfer_contextp)
1253                         *per_transfer_contextp =
1254                                 src_ring->per_transfer_context[sw_index];
1255
1256                 /* sanity */
1257                 src_ring->per_transfer_context[sw_index] = NULL;
1258
1259                 /* Update sw_index */
1260                 sw_index = CE_RING_IDX_INCR(nentries_mask, sw_index);
1261                 src_ring->sw_index = sw_index;
1262                 ret = 0;
1263         } else {
1264                 ret = -EIO;
1265         }
1266
1267         spin_unlock_bh(&ce->ce_lock);
1268
1269         return ret;
1270 }
1271 EXPORT_SYMBOL(ath10k_ce_cancel_send_next);
1272
1273 int ath10k_ce_completed_send_next(struct ath10k_ce_pipe *ce_state,
1274                                   void **per_transfer_contextp)
1275 {
1276         struct ath10k *ar = ce_state->ar;
1277         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1278         int ret;
1279
1280         spin_lock_bh(&ce->ce_lock);
1281         ret = ath10k_ce_completed_send_next_nolock(ce_state,
1282                                                    per_transfer_contextp);
1283         spin_unlock_bh(&ce->ce_lock);
1284
1285         return ret;
1286 }
1287 EXPORT_SYMBOL(ath10k_ce_completed_send_next);
1288
1289 /*
1290  * Guts of interrupt handler for per-engine interrupts on a particular CE.
1291  *
1292  * Invokes registered callbacks for recv_complete,
1293  * send_complete, and watermarks.
1294  */
1295 void ath10k_ce_per_engine_service(struct ath10k *ar, unsigned int ce_id)
1296 {
1297         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1298         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1299         struct ath10k_hw_ce_host_wm_regs *wm_regs = ar->hw_ce_regs->wm_regs;
1300         u32 ctrl_addr = ce_state->ctrl_addr;
1301
1302         spin_lock_bh(&ce->ce_lock);
1303
1304         /* Clear the copy-complete interrupts that will be handled here. */
1305         ath10k_ce_engine_int_status_clear(ar, ctrl_addr,
1306                                           wm_regs->cc_mask);
1307
1308         spin_unlock_bh(&ce->ce_lock);
1309
1310         if (ce_state->recv_cb)
1311                 ce_state->recv_cb(ce_state);
1312
1313         if (ce_state->send_cb)
1314                 ce_state->send_cb(ce_state);
1315
1316         spin_lock_bh(&ce->ce_lock);
1317
1318         /*
1319          * Misc CE interrupts are not being handled, but still need
1320          * to be cleared.
1321          */
1322         ath10k_ce_engine_int_status_clear(ar, ctrl_addr, wm_regs->wm_mask);
1323
1324         spin_unlock_bh(&ce->ce_lock);
1325 }
1326 EXPORT_SYMBOL(ath10k_ce_per_engine_service);
1327
1328 /*
1329  * Handler for per-engine interrupts on ALL active CEs.
1330  * This is used in cases where the system is sharing a
1331  * single interrput for all CEs
1332  */
1333
1334 void ath10k_ce_per_engine_service_any(struct ath10k *ar)
1335 {
1336         int ce_id;
1337         u32 intr_summary;
1338
1339         intr_summary = ath10k_ce_interrupt_summary(ar);
1340
1341         for (ce_id = 0; intr_summary && (ce_id < CE_COUNT); ce_id++) {
1342                 if (intr_summary & (1 << ce_id))
1343                         intr_summary &= ~(1 << ce_id);
1344                 else
1345                         /* no intr pending on this CE */
1346                         continue;
1347
1348                 ath10k_ce_per_engine_service(ar, ce_id);
1349         }
1350 }
1351 EXPORT_SYMBOL(ath10k_ce_per_engine_service_any);
1352
1353 /*
1354  * Adjust interrupts for the copy complete handler.
1355  * If it's needed for either send or recv, then unmask
1356  * this interrupt; otherwise, mask it.
1357  *
1358  * Called with ce_lock held.
1359  */
1360 static void ath10k_ce_per_engine_handler_adjust(struct ath10k_ce_pipe *ce_state)
1361 {
1362         u32 ctrl_addr = ce_state->ctrl_addr;
1363         struct ath10k *ar = ce_state->ar;
1364         bool disable_copy_compl_intr = ce_state->attr_flags & CE_ATTR_DIS_INTR;
1365
1366         if ((!disable_copy_compl_intr) &&
1367             (ce_state->send_cb || ce_state->recv_cb))
1368                 ath10k_ce_copy_complete_inter_enable(ar, ctrl_addr);
1369         else
1370                 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1371
1372         ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1373 }
1374
1375 int ath10k_ce_disable_interrupts(struct ath10k *ar)
1376 {
1377         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1378         struct ath10k_ce_pipe *ce_state;
1379         u32 ctrl_addr;
1380         int ce_id;
1381
1382         for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1383                 ce_state  = &ce->ce_states[ce_id];
1384                 if (ce_state->attr_flags & CE_ATTR_POLL)
1385                         continue;
1386
1387                 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1388
1389                 ath10k_ce_copy_complete_intr_disable(ar, ctrl_addr);
1390                 ath10k_ce_error_intr_disable(ar, ctrl_addr);
1391                 ath10k_ce_watermark_intr_disable(ar, ctrl_addr);
1392         }
1393
1394         return 0;
1395 }
1396 EXPORT_SYMBOL(ath10k_ce_disable_interrupts);
1397
1398 void ath10k_ce_enable_interrupts(struct ath10k *ar)
1399 {
1400         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1401         int ce_id;
1402         struct ath10k_ce_pipe *ce_state;
1403
1404         /* Enable interrupts for copy engine that
1405          * are not using polling mode.
1406          */
1407         for (ce_id = 0; ce_id < CE_COUNT; ce_id++) {
1408                 ce_state  = &ce->ce_states[ce_id];
1409                 if (ce_state->attr_flags & CE_ATTR_POLL)
1410                         continue;
1411
1412                 ath10k_ce_per_engine_handler_adjust(ce_state);
1413         }
1414 }
1415 EXPORT_SYMBOL(ath10k_ce_enable_interrupts);
1416
1417 static int ath10k_ce_init_src_ring(struct ath10k *ar,
1418                                    unsigned int ce_id,
1419                                    const struct ce_attr *attr)
1420 {
1421         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1422         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1423         struct ath10k_ce_ring *src_ring = ce_state->src_ring;
1424         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1425
1426         nentries = roundup_pow_of_two(attr->src_nentries);
1427
1428         if (ar->hw_params.target_64bit)
1429                 memset(src_ring->base_addr_owner_space, 0,
1430                        nentries * sizeof(struct ce_desc_64));
1431         else
1432                 memset(src_ring->base_addr_owner_space, 0,
1433                        nentries * sizeof(struct ce_desc));
1434
1435         src_ring->sw_index = ath10k_ce_src_ring_read_index_get(ar, ctrl_addr);
1436         src_ring->sw_index &= src_ring->nentries_mask;
1437         src_ring->hw_index = src_ring->sw_index;
1438
1439         src_ring->write_index =
1440                 ath10k_ce_src_ring_write_index_get(ar, ctrl_addr);
1441         src_ring->write_index &= src_ring->nentries_mask;
1442
1443         ath10k_ce_src_ring_base_addr_set(ar, ce_id,
1444                                          src_ring->base_addr_ce_space);
1445         ath10k_ce_src_ring_size_set(ar, ctrl_addr, nentries);
1446         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, attr->src_sz_max);
1447         ath10k_ce_src_ring_byte_swap_set(ar, ctrl_addr, 0);
1448         ath10k_ce_src_ring_lowmark_set(ar, ctrl_addr, 0);
1449         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, nentries);
1450
1451         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1452                    "boot init ce src ring id %d entries %d base_addr %pK\n",
1453                    ce_id, nentries, src_ring->base_addr_owner_space);
1454
1455         return 0;
1456 }
1457
1458 static int ath10k_ce_init_dest_ring(struct ath10k *ar,
1459                                     unsigned int ce_id,
1460                                     const struct ce_attr *attr)
1461 {
1462         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1463         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1464         struct ath10k_ce_ring *dest_ring = ce_state->dest_ring;
1465         u32 nentries, ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1466
1467         nentries = roundup_pow_of_two(attr->dest_nentries);
1468
1469         if (ar->hw_params.target_64bit)
1470                 memset(dest_ring->base_addr_owner_space, 0,
1471                        nentries * sizeof(struct ce_desc_64));
1472         else
1473                 memset(dest_ring->base_addr_owner_space, 0,
1474                        nentries * sizeof(struct ce_desc));
1475
1476         dest_ring->sw_index = ath10k_ce_dest_ring_read_index_get(ar, ctrl_addr);
1477         dest_ring->sw_index &= dest_ring->nentries_mask;
1478         dest_ring->write_index =
1479                 ath10k_ce_dest_ring_write_index_get(ar, ctrl_addr);
1480         dest_ring->write_index &= dest_ring->nentries_mask;
1481
1482         ath10k_ce_dest_ring_base_addr_set(ar, ce_id,
1483                                           dest_ring->base_addr_ce_space);
1484         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, nentries);
1485         ath10k_ce_dest_ring_byte_swap_set(ar, ctrl_addr, 0);
1486         ath10k_ce_dest_ring_lowmark_set(ar, ctrl_addr, 0);
1487         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, nentries);
1488
1489         ath10k_dbg(ar, ATH10K_DBG_BOOT,
1490                    "boot ce dest ring id %d entries %d base_addr %pK\n",
1491                    ce_id, nentries, dest_ring->base_addr_owner_space);
1492
1493         return 0;
1494 }
1495
1496 static int ath10k_ce_alloc_shadow_base(struct ath10k *ar,
1497                                        struct ath10k_ce_ring *src_ring,
1498                                        u32 nentries)
1499 {
1500         src_ring->shadow_base_unaligned = kcalloc(nentries,
1501                                                   sizeof(struct ce_desc_64),
1502                                                   GFP_KERNEL);
1503         if (!src_ring->shadow_base_unaligned)
1504                 return -ENOMEM;
1505
1506         src_ring->shadow_base = (struct ce_desc_64 *)
1507                         PTR_ALIGN(src_ring->shadow_base_unaligned,
1508                                   CE_DESC_RING_ALIGN);
1509         return 0;
1510 }
1511
1512 static struct ath10k_ce_ring *
1513 ath10k_ce_alloc_src_ring(struct ath10k *ar, unsigned int ce_id,
1514                          const struct ce_attr *attr)
1515 {
1516         struct ath10k_ce_ring *src_ring;
1517         u32 nentries = attr->src_nentries;
1518         dma_addr_t base_addr;
1519         int ret;
1520
1521         nentries = roundup_pow_of_two(nentries);
1522
1523         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1524                                        nentries), GFP_KERNEL);
1525         if (src_ring == NULL)
1526                 return ERR_PTR(-ENOMEM);
1527
1528         src_ring->nentries = nentries;
1529         src_ring->nentries_mask = nentries - 1;
1530
1531         /*
1532          * Legacy platforms that do not support cache
1533          * coherent DMA are unsupported
1534          */
1535         src_ring->base_addr_owner_space_unaligned =
1536                 dma_alloc_coherent(ar->dev,
1537                                    (nentries * sizeof(struct ce_desc) +
1538                                     CE_DESC_RING_ALIGN),
1539                                    &base_addr, GFP_KERNEL);
1540         if (!src_ring->base_addr_owner_space_unaligned) {
1541                 kfree(src_ring);
1542                 return ERR_PTR(-ENOMEM);
1543         }
1544
1545         src_ring->base_addr_ce_space_unaligned = base_addr;
1546
1547         src_ring->base_addr_owner_space =
1548                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1549                                   CE_DESC_RING_ALIGN);
1550         src_ring->base_addr_ce_space =
1551                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1552                               CE_DESC_RING_ALIGN);
1553
1554         if (ar->hw_params.shadow_reg_support) {
1555                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1556                 if (ret) {
1557                         dma_free_coherent(ar->dev,
1558                                           (nentries * sizeof(struct ce_desc_64) +
1559                                            CE_DESC_RING_ALIGN),
1560                                           src_ring->base_addr_owner_space_unaligned,
1561                                           base_addr);
1562                         kfree(src_ring);
1563                         return ERR_PTR(ret);
1564                 }
1565         }
1566
1567         return src_ring;
1568 }
1569
1570 static struct ath10k_ce_ring *
1571 ath10k_ce_alloc_src_ring_64(struct ath10k *ar, unsigned int ce_id,
1572                             const struct ce_attr *attr)
1573 {
1574         struct ath10k_ce_ring *src_ring;
1575         u32 nentries = attr->src_nentries;
1576         dma_addr_t base_addr;
1577         int ret;
1578
1579         nentries = roundup_pow_of_two(nentries);
1580
1581         src_ring = kzalloc(struct_size(src_ring, per_transfer_context,
1582                                        nentries), GFP_KERNEL);
1583         if (!src_ring)
1584                 return ERR_PTR(-ENOMEM);
1585
1586         src_ring->nentries = nentries;
1587         src_ring->nentries_mask = nentries - 1;
1588
1589         /* Legacy platforms that do not support cache
1590          * coherent DMA are unsupported
1591          */
1592         src_ring->base_addr_owner_space_unaligned =
1593                 dma_alloc_coherent(ar->dev,
1594                                    (nentries * sizeof(struct ce_desc_64) +
1595                                     CE_DESC_RING_ALIGN),
1596                                    &base_addr, GFP_KERNEL);
1597         if (!src_ring->base_addr_owner_space_unaligned) {
1598                 kfree(src_ring);
1599                 return ERR_PTR(-ENOMEM);
1600         }
1601
1602         src_ring->base_addr_ce_space_unaligned = base_addr;
1603
1604         src_ring->base_addr_owner_space =
1605                         PTR_ALIGN(src_ring->base_addr_owner_space_unaligned,
1606                                   CE_DESC_RING_ALIGN);
1607         src_ring->base_addr_ce_space =
1608                         ALIGN(src_ring->base_addr_ce_space_unaligned,
1609                               CE_DESC_RING_ALIGN);
1610
1611         if (ar->hw_params.shadow_reg_support) {
1612                 ret = ath10k_ce_alloc_shadow_base(ar, src_ring, nentries);
1613                 if (ret) {
1614                         dma_free_coherent(ar->dev,
1615                                           (nentries * sizeof(struct ce_desc_64) +
1616                                            CE_DESC_RING_ALIGN),
1617                                           src_ring->base_addr_owner_space_unaligned,
1618                                           base_addr);
1619                         kfree(src_ring);
1620                         return ERR_PTR(ret);
1621                 }
1622         }
1623
1624         return src_ring;
1625 }
1626
1627 static struct ath10k_ce_ring *
1628 ath10k_ce_alloc_dest_ring(struct ath10k *ar, unsigned int ce_id,
1629                           const struct ce_attr *attr)
1630 {
1631         struct ath10k_ce_ring *dest_ring;
1632         u32 nentries;
1633         dma_addr_t base_addr;
1634
1635         nentries = roundup_pow_of_two(attr->dest_nentries);
1636
1637         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1638                                         nentries), GFP_KERNEL);
1639         if (dest_ring == NULL)
1640                 return ERR_PTR(-ENOMEM);
1641
1642         dest_ring->nentries = nentries;
1643         dest_ring->nentries_mask = nentries - 1;
1644
1645         /*
1646          * Legacy platforms that do not support cache
1647          * coherent DMA are unsupported
1648          */
1649         dest_ring->base_addr_owner_space_unaligned =
1650                 dma_alloc_coherent(ar->dev,
1651                                    (nentries * sizeof(struct ce_desc) +
1652                                     CE_DESC_RING_ALIGN),
1653                                    &base_addr, GFP_KERNEL);
1654         if (!dest_ring->base_addr_owner_space_unaligned) {
1655                 kfree(dest_ring);
1656                 return ERR_PTR(-ENOMEM);
1657         }
1658
1659         dest_ring->base_addr_ce_space_unaligned = base_addr;
1660
1661         dest_ring->base_addr_owner_space =
1662                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1663                                   CE_DESC_RING_ALIGN);
1664         dest_ring->base_addr_ce_space =
1665                                 ALIGN(dest_ring->base_addr_ce_space_unaligned,
1666                                       CE_DESC_RING_ALIGN);
1667
1668         return dest_ring;
1669 }
1670
1671 static struct ath10k_ce_ring *
1672 ath10k_ce_alloc_dest_ring_64(struct ath10k *ar, unsigned int ce_id,
1673                              const struct ce_attr *attr)
1674 {
1675         struct ath10k_ce_ring *dest_ring;
1676         u32 nentries;
1677         dma_addr_t base_addr;
1678
1679         nentries = roundup_pow_of_two(attr->dest_nentries);
1680
1681         dest_ring = kzalloc(struct_size(dest_ring, per_transfer_context,
1682                                         nentries), GFP_KERNEL);
1683         if (!dest_ring)
1684                 return ERR_PTR(-ENOMEM);
1685
1686         dest_ring->nentries = nentries;
1687         dest_ring->nentries_mask = nentries - 1;
1688
1689         /* Legacy platforms that do not support cache
1690          * coherent DMA are unsupported
1691          */
1692         dest_ring->base_addr_owner_space_unaligned =
1693                 dma_alloc_coherent(ar->dev,
1694                                    (nentries * sizeof(struct ce_desc_64) +
1695                                     CE_DESC_RING_ALIGN),
1696                                    &base_addr, GFP_KERNEL);
1697         if (!dest_ring->base_addr_owner_space_unaligned) {
1698                 kfree(dest_ring);
1699                 return ERR_PTR(-ENOMEM);
1700         }
1701
1702         dest_ring->base_addr_ce_space_unaligned = base_addr;
1703
1704         /* Correctly initialize memory to 0 to prevent garbage
1705          * data crashing system when download firmware
1706          */
1707         memset(dest_ring->base_addr_owner_space_unaligned, 0,
1708                nentries * sizeof(struct ce_desc_64) + CE_DESC_RING_ALIGN);
1709
1710         dest_ring->base_addr_owner_space =
1711                         PTR_ALIGN(dest_ring->base_addr_owner_space_unaligned,
1712                                   CE_DESC_RING_ALIGN);
1713         dest_ring->base_addr_ce_space =
1714                         ALIGN(dest_ring->base_addr_ce_space_unaligned,
1715                               CE_DESC_RING_ALIGN);
1716
1717         return dest_ring;
1718 }
1719
1720 /*
1721  * Initialize a Copy Engine based on caller-supplied attributes.
1722  * This may be called once to initialize both source and destination
1723  * rings or it may be called twice for separate source and destination
1724  * initialization. It may be that only one side or the other is
1725  * initialized by software/firmware.
1726  */
1727 int ath10k_ce_init_pipe(struct ath10k *ar, unsigned int ce_id,
1728                         const struct ce_attr *attr)
1729 {
1730         int ret;
1731
1732         if (attr->src_nentries) {
1733                 ret = ath10k_ce_init_src_ring(ar, ce_id, attr);
1734                 if (ret) {
1735                         ath10k_err(ar, "Failed to initialize CE src ring for ID: %d (%d)\n",
1736                                    ce_id, ret);
1737                         return ret;
1738                 }
1739         }
1740
1741         if (attr->dest_nentries) {
1742                 ret = ath10k_ce_init_dest_ring(ar, ce_id, attr);
1743                 if (ret) {
1744                         ath10k_err(ar, "Failed to initialize CE dest ring for ID: %d (%d)\n",
1745                                    ce_id, ret);
1746                         return ret;
1747                 }
1748         }
1749
1750         return 0;
1751 }
1752 EXPORT_SYMBOL(ath10k_ce_init_pipe);
1753
1754 static void ath10k_ce_deinit_src_ring(struct ath10k *ar, unsigned int ce_id)
1755 {
1756         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1757
1758         ath10k_ce_src_ring_base_addr_set(ar, ce_id, 0);
1759         ath10k_ce_src_ring_size_set(ar, ctrl_addr, 0);
1760         ath10k_ce_src_ring_dmax_set(ar, ctrl_addr, 0);
1761         ath10k_ce_src_ring_highmark_set(ar, ctrl_addr, 0);
1762 }
1763
1764 static void ath10k_ce_deinit_dest_ring(struct ath10k *ar, unsigned int ce_id)
1765 {
1766         u32 ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1767
1768         ath10k_ce_dest_ring_base_addr_set(ar, ce_id, 0);
1769         ath10k_ce_dest_ring_size_set(ar, ctrl_addr, 0);
1770         ath10k_ce_dest_ring_highmark_set(ar, ctrl_addr, 0);
1771 }
1772
1773 void ath10k_ce_deinit_pipe(struct ath10k *ar, unsigned int ce_id)
1774 {
1775         ath10k_ce_deinit_src_ring(ar, ce_id);
1776         ath10k_ce_deinit_dest_ring(ar, ce_id);
1777 }
1778 EXPORT_SYMBOL(ath10k_ce_deinit_pipe);
1779
1780 static void _ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1781 {
1782         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1783         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1784
1785         if (ce_state->src_ring) {
1786                 if (ar->hw_params.shadow_reg_support)
1787                         kfree(ce_state->src_ring->shadow_base_unaligned);
1788                 dma_free_coherent(ar->dev,
1789                                   (ce_state->src_ring->nentries *
1790                                    sizeof(struct ce_desc) +
1791                                    CE_DESC_RING_ALIGN),
1792                                   ce_state->src_ring->base_addr_owner_space,
1793                                   ce_state->src_ring->base_addr_ce_space);
1794                 kfree(ce_state->src_ring);
1795         }
1796
1797         if (ce_state->dest_ring) {
1798                 dma_free_coherent(ar->dev,
1799                                   (ce_state->dest_ring->nentries *
1800                                    sizeof(struct ce_desc) +
1801                                    CE_DESC_RING_ALIGN),
1802                                   ce_state->dest_ring->base_addr_owner_space,
1803                                   ce_state->dest_ring->base_addr_ce_space);
1804                 kfree(ce_state->dest_ring);
1805         }
1806
1807         ce_state->src_ring = NULL;
1808         ce_state->dest_ring = NULL;
1809 }
1810
1811 static void _ath10k_ce_free_pipe_64(struct ath10k *ar, int ce_id)
1812 {
1813         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1814         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1815
1816         if (ce_state->src_ring) {
1817                 if (ar->hw_params.shadow_reg_support)
1818                         kfree(ce_state->src_ring->shadow_base_unaligned);
1819                 dma_free_coherent(ar->dev,
1820                                   (ce_state->src_ring->nentries *
1821                                    sizeof(struct ce_desc_64) +
1822                                    CE_DESC_RING_ALIGN),
1823                                   ce_state->src_ring->base_addr_owner_space,
1824                                   ce_state->src_ring->base_addr_ce_space);
1825                 kfree(ce_state->src_ring);
1826         }
1827
1828         if (ce_state->dest_ring) {
1829                 dma_free_coherent(ar->dev,
1830                                   (ce_state->dest_ring->nentries *
1831                                    sizeof(struct ce_desc_64) +
1832                                    CE_DESC_RING_ALIGN),
1833                                   ce_state->dest_ring->base_addr_owner_space,
1834                                   ce_state->dest_ring->base_addr_ce_space);
1835                 kfree(ce_state->dest_ring);
1836         }
1837
1838         ce_state->src_ring = NULL;
1839         ce_state->dest_ring = NULL;
1840 }
1841
1842 void ath10k_ce_free_pipe(struct ath10k *ar, int ce_id)
1843 {
1844         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1845         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1846
1847         ce_state->ops->ce_free_pipe(ar, ce_id);
1848 }
1849 EXPORT_SYMBOL(ath10k_ce_free_pipe);
1850
1851 void ath10k_ce_dump_registers(struct ath10k *ar,
1852                               struct ath10k_fw_crash_data *crash_data)
1853 {
1854         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1855         struct ath10k_ce_crash_data ce_data;
1856         u32 addr, id;
1857
1858         lockdep_assert_held(&ar->data_lock);
1859
1860         ath10k_err(ar, "Copy Engine register dump:\n");
1861
1862         spin_lock_bh(&ce->ce_lock);
1863         for (id = 0; id < CE_COUNT; id++) {
1864                 addr = ath10k_ce_base_address(ar, id);
1865                 ce_data.base_addr = cpu_to_le32(addr);
1866
1867                 ce_data.src_wr_idx =
1868                         cpu_to_le32(ath10k_ce_src_ring_write_index_get(ar, addr));
1869                 ce_data.src_r_idx =
1870                         cpu_to_le32(ath10k_ce_src_ring_read_index_get(ar, addr));
1871                 ce_data.dst_wr_idx =
1872                         cpu_to_le32(ath10k_ce_dest_ring_write_index_get(ar, addr));
1873                 ce_data.dst_r_idx =
1874                         cpu_to_le32(ath10k_ce_dest_ring_read_index_get(ar, addr));
1875
1876                 if (crash_data)
1877                         crash_data->ce_crash_data[id] = ce_data;
1878
1879                 ath10k_err(ar, "[%02d]: 0x%08x %3u %3u %3u %3u", id,
1880                            le32_to_cpu(ce_data.base_addr),
1881                            le32_to_cpu(ce_data.src_wr_idx),
1882                            le32_to_cpu(ce_data.src_r_idx),
1883                            le32_to_cpu(ce_data.dst_wr_idx),
1884                            le32_to_cpu(ce_data.dst_r_idx));
1885         }
1886
1887         spin_unlock_bh(&ce->ce_lock);
1888 }
1889 EXPORT_SYMBOL(ath10k_ce_dump_registers);
1890
1891 static const struct ath10k_ce_ops ce_ops = {
1892         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring,
1893         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring,
1894         .ce_rx_post_buf = __ath10k_ce_rx_post_buf,
1895         .ce_completed_recv_next_nolock = _ath10k_ce_completed_recv_next_nolock,
1896         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next,
1897         .ce_extract_desc_data = ath10k_ce_extract_desc_data,
1898         .ce_free_pipe = _ath10k_ce_free_pipe,
1899         .ce_send_nolock = _ath10k_ce_send_nolock,
1900         .ce_set_src_ring_base_addr_hi = NULL,
1901         .ce_set_dest_ring_base_addr_hi = NULL,
1902         .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock,
1903 };
1904
1905 static const struct ath10k_ce_ops ce_64_ops = {
1906         .ce_alloc_src_ring = ath10k_ce_alloc_src_ring_64,
1907         .ce_alloc_dst_ring = ath10k_ce_alloc_dest_ring_64,
1908         .ce_rx_post_buf = __ath10k_ce_rx_post_buf_64,
1909         .ce_completed_recv_next_nolock =
1910                                 _ath10k_ce_completed_recv_next_nolock_64,
1911         .ce_revoke_recv_next = _ath10k_ce_revoke_recv_next_64,
1912         .ce_extract_desc_data = ath10k_ce_extract_desc_data_64,
1913         .ce_free_pipe = _ath10k_ce_free_pipe_64,
1914         .ce_send_nolock = _ath10k_ce_send_nolock_64,
1915         .ce_set_src_ring_base_addr_hi = ath10k_ce_set_src_ring_base_addr_hi,
1916         .ce_set_dest_ring_base_addr_hi = ath10k_ce_set_dest_ring_base_addr_hi,
1917         .ce_completed_send_next_nolock = _ath10k_ce_completed_send_next_nolock_64,
1918 };
1919
1920 static void ath10k_ce_set_ops(struct ath10k *ar,
1921                               struct ath10k_ce_pipe *ce_state)
1922 {
1923         switch (ar->hw_rev) {
1924         case ATH10K_HW_WCN3990:
1925                 ce_state->ops = &ce_64_ops;
1926                 break;
1927         default:
1928                 ce_state->ops = &ce_ops;
1929                 break;
1930         }
1931 }
1932
1933 int ath10k_ce_alloc_pipe(struct ath10k *ar, int ce_id,
1934                          const struct ce_attr *attr)
1935 {
1936         struct ath10k_ce *ce = ath10k_ce_priv(ar);
1937         struct ath10k_ce_pipe *ce_state = &ce->ce_states[ce_id];
1938         int ret;
1939
1940         ath10k_ce_set_ops(ar, ce_state);
1941         /* Make sure there's enough CE ringbuffer entries for HTT TX to avoid
1942          * additional TX locking checks.
1943          *
1944          * For the lack of a better place do the check here.
1945          */
1946         BUILD_BUG_ON(2 * TARGET_NUM_MSDU_DESC >
1947                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1948         BUILD_BUG_ON(2 * TARGET_10_4_NUM_MSDU_DESC_PFC >
1949                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1950         BUILD_BUG_ON(2 * TARGET_TLV_NUM_MSDU_DESC >
1951                      (CE_HTT_H2T_MSG_SRC_NENTRIES - 1));
1952
1953         ce_state->ar = ar;
1954         ce_state->id = ce_id;
1955         ce_state->ctrl_addr = ath10k_ce_base_address(ar, ce_id);
1956         ce_state->attr_flags = attr->flags;
1957         ce_state->src_sz_max = attr->src_sz_max;
1958
1959         if (attr->src_nentries)
1960                 ce_state->send_cb = attr->send_cb;
1961
1962         if (attr->dest_nentries)
1963                 ce_state->recv_cb = attr->recv_cb;
1964
1965         if (attr->src_nentries) {
1966                 ce_state->src_ring =
1967                         ce_state->ops->ce_alloc_src_ring(ar, ce_id, attr);
1968                 if (IS_ERR(ce_state->src_ring)) {
1969                         ret = PTR_ERR(ce_state->src_ring);
1970                         ath10k_err(ar, "failed to alloc CE src ring %d: %d\n",
1971                                    ce_id, ret);
1972                         ce_state->src_ring = NULL;
1973                         return ret;
1974                 }
1975         }
1976
1977         if (attr->dest_nentries) {
1978                 ce_state->dest_ring = ce_state->ops->ce_alloc_dst_ring(ar,
1979                                                                         ce_id,
1980                                                                         attr);
1981                 if (IS_ERR(ce_state->dest_ring)) {
1982                         ret = PTR_ERR(ce_state->dest_ring);
1983                         ath10k_err(ar, "failed to alloc CE dest ring %d: %d\n",
1984                                    ce_id, ret);
1985                         ce_state->dest_ring = NULL;
1986                         return ret;
1987                 }
1988         }
1989
1990         return 0;
1991 }
1992 EXPORT_SYMBOL(ath10k_ce_alloc_pipe);
1993
1994 void ath10k_ce_alloc_rri(struct ath10k *ar)
1995 {
1996         int i;
1997         u32 value;
1998         u32 ctrl1_regs;
1999         u32 ce_base_addr;
2000         struct ath10k_ce *ce = ath10k_ce_priv(ar);
2001
2002         ce->vaddr_rri = dma_alloc_coherent(ar->dev,
2003                                            (CE_COUNT * sizeof(u32)),
2004                                            &ce->paddr_rri, GFP_KERNEL);
2005
2006         if (!ce->vaddr_rri)
2007                 return;
2008
2009         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_low,
2010                           lower_32_bits(ce->paddr_rri));
2011         ath10k_ce_write32(ar, ar->hw_ce_regs->ce_rri_high,
2012                           (upper_32_bits(ce->paddr_rri) &
2013                           CE_DESC_ADDR_HI_MASK));
2014
2015         for (i = 0; i < CE_COUNT; i++) {
2016                 ctrl1_regs = ar->hw_ce_regs->ctrl1_regs->addr;
2017                 ce_base_addr = ath10k_ce_base_address(ar, i);
2018                 value = ath10k_ce_read32(ar, ce_base_addr + ctrl1_regs);
2019                 value |= ar->hw_ce_regs->upd->mask;
2020                 ath10k_ce_write32(ar, ce_base_addr + ctrl1_regs, value);
2021         }
2022
2023         memset(ce->vaddr_rri, 0, CE_COUNT * sizeof(u32));
2024 }
2025 EXPORT_SYMBOL(ath10k_ce_alloc_rri);
2026
2027 void ath10k_ce_free_rri(struct ath10k *ar)
2028 {
2029         struct ath10k_ce *ce = ath10k_ce_priv(ar);
2030
2031         dma_free_coherent(ar->dev, (CE_COUNT * sizeof(u32)),
2032                           ce->vaddr_rri,
2033                           ce->paddr_rri);
2034 }
2035 EXPORT_SYMBOL(ath10k_ce_free_rri);