drivers/infiniband/hw/mlx5/odp.c

   1 /*
   2  * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved.
   3  *
   4  * This software is available to you under a choice of one of two
   5  * licenses.  You may choose to be licensed under the terms of the GNU
   6  * General Public License (GPL) Version 2, available from the file
   7  * COPYING in the main directory of this source tree, or the
   8  * OpenIB.org BSD license below:
   9  *
  10  *     Redistribution and use in source and binary forms, with or
  11  *     without modification, are permitted provided that the following
  12  *     conditions are met:
  13  *
  14  *      - Redistributions of source code must retain the above
  15  *        copyright notice, this list of conditions and the following
  16  *        disclaimer.
  17  *
  18  *      - Redistributions in binary form must reproduce the above
  19  *        copyright notice, this list of conditions and the following
  20  *        disclaimer in the documentation and/or other materials
  21  *        provided with the distribution.
  22  *
  23  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26  * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27  * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28  * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29  * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30  * SOFTWARE.
  31  */
  32
  33 #include <rdma/ib_umem.h>
  34 #include <rdma/ib_umem_odp.h>
  35 #include <linux/kernel.h>
  36
  37 #include "mlx5_ib.h"
  38 #include "cmd.h"
  39
  40 #include <linux/mlx5/eq.h>
  41
  42 /* Contains the details of a pagefault. */
  43 struct mlx5_pagefault {
  44         u32                     bytes_committed;
  45         u32                     token;
  46         u8                      event_subtype;
  47         u8                      type;
  48         union {
  49                 /* Initiator or send message responder pagefault details. */
  50                 struct {
  51                         /* Received packet size, only valid for responders. */
  52                         u32     packet_size;
  53                         /*
  54                          * Number of resource holding WQE, depends on type.
  55                          */
  56                         u32     wq_num;
  57                         /*
  58                          * WQE index. Refers to either the send queue or
  59                          * receive queue, according to event_subtype.
  60                          */
  61                         u16     wqe_index;
  62                 } wqe;
  63                 /* RDMA responder pagefault details */
  64                 struct {
  65                         u32     r_key;
  66                         /*
  67                          * Received packet size, minimal size page fault
  68                          * resolution required for forward progress.
  69                          */
  70                         u32     packet_size;
  71                         u32     rdma_op_len;
  72                         u64     rdma_va;
  73                 } rdma;
  74         };
  75
  76         struct mlx5_ib_pf_eq    *eq;
  77         struct work_struct      work;
  78 };
  79
  80 #define MAX_PREFETCH_LEN (4*1024*1024U)
  81
  82 /* Timeout in ms to wait for an active mmu notifier to complete when handling
  83  * a pagefault. */
  84 #define MMU_NOTIFIER_TIMEOUT 1000
  85
  86 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT)
  87 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT)
  88 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS)
  89 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT)
  90 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1))
  91
  92 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT
  93
  94 static u64 mlx5_imr_ksm_entries;
  95
  96 static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries,
  97                         struct mlx5_ib_mr *imr, int flags)
  98 {
  99         struct mlx5_klm *end = pklm + nentries;
 100
 101         if (flags & MLX5_IB_UPD_XLT_ZAP) {
 102                 for (; pklm != end; pklm++, idx++) {
 103                         pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
 104                         pklm->key = cpu_to_be32(imr->dev->null_mkey);
 105                         pklm->va = 0;
 106                 }
 107                 return;
 108         }
 109
 110         /*
 111          * The locking here is pretty subtle. Ideally the implicit_children
 112          * xarray would be protected by the umem_mutex, however that is not
 113          * possible. Instead this uses a weaker update-then-lock pattern:
 114          *
 115          *  srcu_read_lock()
 116          *    xa_store()
 117          *    mutex_lock(umem_mutex)
 118          *     mlx5_ib_update_xlt()
 119          *    mutex_unlock(umem_mutex)
 120          *    destroy lkey
 121          *
 122          * ie any change the xarray must be followed by the locked update_xlt
 123          * before destroying.
 124          *
 125          * The umem_mutex provides the acquire/release semantic needed to make
 126          * the xa_store() visible to a racing thread. While SRCU is not
 127          * technically required, using it gives consistent use of the SRCU
 128          * locking around the xarray.
 129          */
 130         lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex);
 131         lockdep_assert_held(&imr->dev->odp_srcu);
 132
 133         for (; pklm != end; pklm++, idx++) {
 134                 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx);
 135
 136                 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE);
 137                 if (mtt) {
 138                         pklm->key = cpu_to_be32(mtt->ibmr.lkey);
 139                         pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE);
 140                 } else {
 141                         pklm->key = cpu_to_be32(imr->dev->null_mkey);
 142                         pklm->va = 0;
 143                 }
 144         }
 145 }
 146
 147 static u64 umem_dma_to_mtt(dma_addr_t umem_dma)
 148 {
 149         u64 mtt_entry = umem_dma & ODP_DMA_ADDR_MASK;
 150
 151         if (umem_dma & ODP_READ_ALLOWED_BIT)
 152                 mtt_entry |= MLX5_IB_MTT_READ;
 153         if (umem_dma & ODP_WRITE_ALLOWED_BIT)
 154                 mtt_entry |= MLX5_IB_MTT_WRITE;
 155
 156         return mtt_entry;
 157 }
 158
 159 static void populate_mtt(__be64 *pas, size_t idx, size_t nentries,
 160                          struct mlx5_ib_mr *mr, int flags)
 161 {
 162         struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
 163         dma_addr_t pa;
 164         size_t i;
 165
 166         if (flags & MLX5_IB_UPD_XLT_ZAP)
 167                 return;
 168
 169         for (i = 0; i < nentries; i++) {
 170                 pa = odp->dma_list[idx + i];
 171                 pas[i] = cpu_to_be64(umem_dma_to_mtt(pa));
 172         }
 173 }
 174
 175 void mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries,
 176                            struct mlx5_ib_mr *mr, int flags)
 177 {
 178         if (flags & MLX5_IB_UPD_XLT_INDIRECT) {
 179                 populate_klm(xlt, idx, nentries, mr, flags);
 180         } else {
 181                 populate_mtt(xlt, idx, nentries, mr, flags);
 182         }
 183 }
 184
 185 static void dma_fence_odp_mr(struct mlx5_ib_mr *mr)
 186 {
 187         struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
 188
 189         /* Ensure mlx5_ib_invalidate_range() will not touch the MR any more */
 190         mutex_lock(&odp->umem_mutex);
 191         if (odp->npages) {
 192                 mlx5_mr_cache_invalidate(mr);
 193                 ib_umem_odp_unmap_dma_pages(odp, ib_umem_start(odp),
 194                                             ib_umem_end(odp));
 195                 WARN_ON(odp->npages);
 196         }
 197         odp->private = NULL;
 198         mutex_unlock(&odp->umem_mutex);
 199
 200         if (!mr->allocated_from_cache) {
 201                 mlx5_core_destroy_mkey(mr->dev->mdev, &mr->mmkey);
 202                 WARN_ON(mr->descs);
 203         }
 204 }
 205
 206 /*
 207  * This must be called after the mr has been removed from implicit_children
 208  * and the SRCU synchronized.  NOTE: The MR does not necessarily have to be
 209  * empty here, parallel page faults could have raced with the free process and
 210  * added pages to it.
 211  */
 212 static void free_implicit_child_mr(struct mlx5_ib_mr *mr, bool need_imr_xlt)
 213 {
 214         struct mlx5_ib_mr *imr = mr->parent;
 215         struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);
 216         struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
 217         unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT;
 218         int srcu_key;
 219
 220         /* implicit_child_mr's are not allowed to have deferred work */
 221         WARN_ON(atomic_read(&mr->num_deferred_work));
 222
 223         if (need_imr_xlt) {
 224                 srcu_key = srcu_read_lock(&mr->dev->odp_srcu);
 225                 mutex_lock(&odp_imr->umem_mutex);
 226                 mlx5_ib_update_xlt(mr->parent, idx, 1, 0,
 227                                    MLX5_IB_UPD_XLT_INDIRECT |
 228                                    MLX5_IB_UPD_XLT_ATOMIC);
 229                 mutex_unlock(&odp_imr->umem_mutex);
 230                 srcu_read_unlock(&mr->dev->odp_srcu, srcu_key);
 231         }
 232
 233         dma_fence_odp_mr(mr);
 234
 235         mr->parent = NULL;
 236         mlx5_mr_cache_free(mr->dev, mr);
 237         ib_umem_odp_release(odp);
 238         atomic_dec(&imr->num_deferred_work);
 239 }
 240
 241 static void free_implicit_child_mr_work(struct work_struct *work)
 242 {
 243         struct mlx5_ib_mr *mr =
 244                 container_of(work, struct mlx5_ib_mr, odp_destroy.work);
 245
 246         free_implicit_child_mr(mr, true);
 247 }
 248
 249 static void free_implicit_child_mr_rcu(struct rcu_head *head)
 250 {
 251         struct mlx5_ib_mr *mr =
 252                 container_of(head, struct mlx5_ib_mr, odp_destroy.rcu);
 253
 254         /* Freeing a MR is a sleeping operation, so bounce to a work queue */
 255         INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work);
 256         queue_work(system_unbound_wq, &mr->odp_destroy.work);
 257 }
 258
 259 static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr)
 260 {
 261         struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
 262         unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT;
 263         struct mlx5_ib_mr *imr = mr->parent;
 264
 265         xa_lock(&imr->implicit_children);
 266         /*
 267          * This can race with mlx5_ib_free_implicit_mr(), the first one to
 268          * reach the xa lock wins the race and destroys the MR.
 269          */
 270         if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_ATOMIC) !=
 271             mr)
 272                 goto out_unlock;
 273
 274         atomic_inc(&imr->num_deferred_work);
 275         call_srcu(&mr->dev->odp_srcu, &mr->odp_destroy.rcu,
 276                   free_implicit_child_mr_rcu);
 277
 278 out_unlock:
 279         xa_unlock(&imr->implicit_children);
 280 }
 281
 282 static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni,
 283                                      const struct mmu_notifier_range *range,
 284                                      unsigned long cur_seq)
 285 {
 286         struct ib_umem_odp *umem_odp =
 287                 container_of(mni, struct ib_umem_odp, notifier);
 288         struct mlx5_ib_mr *mr;
 289         const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT /
 290                                     sizeof(struct mlx5_mtt)) - 1;
 291         u64 idx = 0, blk_start_idx = 0;
 292         u64 invalidations = 0;
 293         unsigned long start;
 294         unsigned long end;
 295         int in_block = 0;
 296         u64 addr;
 297
 298         if (!mmu_notifier_range_blockable(range))
 299                 return false;
 300
 301         mutex_lock(&umem_odp->umem_mutex);
 302         mmu_interval_set_seq(mni, cur_seq);
 303         /*
 304          * If npages is zero then umem_odp->private may not be setup yet. This
 305          * does not complete until after the first page is mapped for DMA.
 306          */
 307         if (!umem_odp->npages)
 308                 goto out;
 309         mr = umem_odp->private;
 310
 311         start = max_t(u64, ib_umem_start(umem_odp), range->start);
 312         end = min_t(u64, ib_umem_end(umem_odp), range->end);
 313
 314         /*
 315          * Iteration one - zap the HW's MTTs. The notifiers_count ensures that
 316          * while we are doing the invalidation, no page fault will attempt to
 317          * overwrite the same MTTs.  Concurent invalidations might race us,
 318          * but they will write 0s as well, so no difference in the end result.
 319          */
 320         for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) {
 321                 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift;
 322                 /*
 323                  * Strive to write the MTTs in chunks, but avoid overwriting
 324                  * non-existing MTTs. The huristic here can be improved to
 325                  * estimate the cost of another UMR vs. the cost of bigger
 326                  * UMR.
 327                  */
 328                 if (umem_odp->dma_list[idx] &
 329                     (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) {
 330                         if (!in_block) {
 331                                 blk_start_idx = idx;
 332                                 in_block = 1;
 333                         }
 334
 335                         /* Count page invalidations */
 336                         invalidations += idx - blk_start_idx + 1;
 337                 } else {
 338                         u64 umr_offset = idx & umr_block_mask;
 339
 340                         if (in_block && umr_offset == 0) {
 341                                 mlx5_ib_update_xlt(mr, blk_start_idx,
 342                                                    idx - blk_start_idx, 0,
 343                                                    MLX5_IB_UPD_XLT_ZAP |
 344                                                    MLX5_IB_UPD_XLT_ATOMIC);
 345                                 in_block = 0;
 346                         }
 347                 }
 348         }
 349         if (in_block)
 350                 mlx5_ib_update_xlt(mr, blk_start_idx,
 351                                    idx - blk_start_idx + 1, 0,
 352                                    MLX5_IB_UPD_XLT_ZAP |
 353                                    MLX5_IB_UPD_XLT_ATOMIC);
 354
 355         mlx5_update_odp_stats(mr, invalidations, invalidations);
 356
 357         /*
 358          * We are now sure that the device will not access the
 359          * memory. We can safely unmap it, and mark it as dirty if
 360          * needed.
 361          */
 362
 363         ib_umem_odp_unmap_dma_pages(umem_odp, start, end);
 364
 365         if (unlikely(!umem_odp->npages && mr->parent))
 366                 destroy_unused_implicit_child_mr(mr);
 367 out:
 368         mutex_unlock(&umem_odp->umem_mutex);
 369         return true;
 370 }
 371
 372 const struct mmu_interval_notifier_ops mlx5_mn_ops = {
 373         .invalidate = mlx5_ib_invalidate_range,
 374 };
 375
 376 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev)
 377 {
 378         struct ib_odp_caps *caps = &dev->odp_caps;
 379
 380         memset(caps, 0, sizeof(*caps));
 381
 382         if (!MLX5_CAP_GEN(dev->mdev, pg) ||
 383             !mlx5_ib_can_use_umr(dev, true, 0))
 384                 return;
 385
 386         caps->general_caps = IB_ODP_SUPPORT;
 387
 388         if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset))
 389                 dev->odp_max_size = U64_MAX;
 390         else
 391                 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT);
 392
 393         if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send))
 394                 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND;
 395
 396         if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.srq_receive))
 397                 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 398
 399         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send))
 400                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND;
 401
 402         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive))
 403                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV;
 404
 405         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write))
 406                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE;
 407
 408         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read))
 409                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ;
 410
 411         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic))
 412                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;
 413
 414         if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.srq_receive))
 415                 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 416
 417         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.send))
 418                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND;
 419
 420         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.receive))
 421                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV;
 422
 423         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.write))
 424                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE;
 425
 426         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.read))
 427                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ;
 428
 429         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.atomic))
 430                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC;
 431
 432         if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.srq_receive))
 433                 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV;
 434
 435         if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) &&
 436             MLX5_CAP_GEN(dev->mdev, null_mkey) &&
 437             MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) &&
 438             !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled))
 439                 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT;
 440 }
 441
 442 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev,
 443                                       struct mlx5_pagefault *pfault,
 444                                       int error)
 445 {
 446         int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ?
 447                      pfault->wqe.wq_num : pfault->token;
 448         u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { };
 449         u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)]   = { };
 450         int err;
 451
 452         MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME);
 453         MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type);
 454         MLX5_SET(page_fault_resume_in, in, token, pfault->token);
 455         MLX5_SET(page_fault_resume_in, in, wq_number, wq_num);
 456         MLX5_SET(page_fault_resume_in, in, error, !!error);
 457
 458         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
 459         if (err)
 460                 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n",
 461                             wq_num, err);
 462 }
 463
 464 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr,
 465                                                 unsigned long idx)
 466 {
 467         struct ib_umem_odp *odp;
 468         struct mlx5_ib_mr *mr;
 469         struct mlx5_ib_mr *ret;
 470         int err;
 471
 472         odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem),
 473                                       idx * MLX5_IMR_MTT_SIZE,
 474                                       MLX5_IMR_MTT_SIZE, &mlx5_mn_ops);
 475         if (IS_ERR(odp))
 476                 return ERR_CAST(odp);
 477
 478         ret = mr = mlx5_mr_cache_alloc(imr->dev, MLX5_IMR_MTT_CACHE_ENTRY);
 479         if (IS_ERR(mr))
 480                 goto out_umem;
 481
 482         mr->ibmr.pd = imr->ibmr.pd;
 483         mr->access_flags = imr->access_flags;
 484         mr->umem = &odp->umem;
 485         mr->ibmr.lkey = mr->mmkey.key;
 486         mr->ibmr.rkey = mr->mmkey.key;
 487         mr->mmkey.iova = idx * MLX5_IMR_MTT_SIZE;
 488         mr->parent = imr;
 489         odp->private = mr;
 490
 491         err = mlx5_ib_update_xlt(mr, 0,
 492                                  MLX5_IMR_MTT_ENTRIES,
 493                                  PAGE_SHIFT,
 494                                  MLX5_IB_UPD_XLT_ZAP |
 495                                  MLX5_IB_UPD_XLT_ENABLE);
 496         if (err) {
 497                 ret = ERR_PTR(err);
 498                 goto out_mr;
 499         }
 500
 501         /*
 502          * Once the store to either xarray completes any error unwind has to
 503          * use synchronize_srcu(). Avoid this with xa_reserve()
 504          */
 505         ret = xa_cmpxchg(&imr->implicit_children, idx, NULL, mr,
 506                          GFP_KERNEL);
 507         if (unlikely(ret)) {
 508                 if (xa_is_err(ret)) {
 509                         ret = ERR_PTR(xa_err(ret));
 510                         goto out_mr;
 511                 }
 512                 /*
 513                  * Another thread beat us to creating the child mr, use
 514                  * theirs.
 515                  */
 516                 goto out_mr;
 517         }
 518
 519         mlx5_ib_dbg(imr->dev, "key %x mr %p\n", mr->mmkey.key, mr);
 520         return mr;
 521
 522 out_mr:
 523         mlx5_mr_cache_free(imr->dev, mr);
 524 out_umem:
 525         ib_umem_odp_release(odp);
 526         return ret;
 527 }
 528
 529 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd,
 530                                              struct ib_udata *udata,
 531                                              int access_flags)
 532 {
 533         struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device);
 534         struct ib_umem_odp *umem_odp;
 535         struct mlx5_ib_mr *imr;
 536         int err;
 537
 538         umem_odp = ib_umem_odp_alloc_implicit(&dev->ib_dev, access_flags);
 539         if (IS_ERR(umem_odp))
 540                 return ERR_CAST(umem_odp);
 541
 542         imr = mlx5_mr_cache_alloc(dev, MLX5_IMR_KSM_CACHE_ENTRY);
 543         if (IS_ERR(imr)) {
 544                 err = PTR_ERR(imr);
 545                 goto out_umem;
 546         }
 547
 548         imr->ibmr.pd = &pd->ibpd;
 549         imr->access_flags = access_flags;
 550         imr->mmkey.iova = 0;
 551         imr->umem = &umem_odp->umem;
 552         imr->ibmr.lkey = imr->mmkey.key;
 553         imr->ibmr.rkey = imr->mmkey.key;
 554         imr->umem = &umem_odp->umem;
 555         imr->is_odp_implicit = true;
 556         atomic_set(&imr->num_deferred_work, 0);
 557         xa_init(&imr->implicit_children);
 558
 559         err = mlx5_ib_update_xlt(imr, 0,
 560                                  mlx5_imr_ksm_entries,
 561                                  MLX5_KSM_PAGE_SHIFT,
 562                                  MLX5_IB_UPD_XLT_INDIRECT |
 563                                  MLX5_IB_UPD_XLT_ZAP |
 564                                  MLX5_IB_UPD_XLT_ENABLE);
 565         if (err)
 566                 goto out_mr;
 567
 568         err = xa_err(xa_store(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key),
 569                               &imr->mmkey, GFP_KERNEL));
 570         if (err)
 571                 goto out_mr;
 572
 573         mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr);
 574         return imr;
 575 out_mr:
 576         mlx5_ib_err(dev, "Failed to register MKEY %d\n", err);
 577         mlx5_mr_cache_free(dev, imr);
 578 out_umem:
 579         ib_umem_odp_release(umem_odp);
 580         return ERR_PTR(err);
 581 }
 582
 583 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr)
 584 {
 585         struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem);
 586         struct mlx5_ib_dev *dev = imr->dev;
 587         struct list_head destroy_list;
 588         struct mlx5_ib_mr *mtt;
 589         struct mlx5_ib_mr *tmp;
 590         unsigned long idx;
 591
 592         INIT_LIST_HEAD(&destroy_list);
 593
 594         xa_erase(&dev->odp_mkeys, mlx5_base_mkey(imr->mmkey.key));
 595         /*
 596          * This stops the SRCU protected page fault path from touching either
 597          * the imr or any children. The page fault path can only reach the
 598          * children xarray via the imr.
 599          */
 600         synchronize_srcu(&dev->odp_srcu);
 601
 602         xa_lock(&imr->implicit_children);
 603         xa_for_each (&imr->implicit_children, idx, mtt) {
 604                 __xa_erase(&imr->implicit_children, idx);
 605                 list_add(&mtt->odp_destroy.elm, &destroy_list);
 606         }
 607         xa_unlock(&imr->implicit_children);
 608
 609         /*
 610          * num_deferred_work can only be incremented inside the odp_srcu, or
 611          * under xa_lock while the child is in the xarray. Thus at this point
 612          * it is only decreasing, and all work holding it is now on the wq.
 613          */
 614         if (atomic_read(&imr->num_deferred_work)) {
 615                 flush_workqueue(system_unbound_wq);
 616                 WARN_ON(atomic_read(&imr->num_deferred_work));
 617         }
 618
 619         /*
 620          * Fence the imr before we destroy the children. This allows us to
 621          * skip updating the XLT of the imr during destroy of the child mkey
 622          * the imr points to.
 623          */
 624         mlx5_mr_cache_invalidate(imr);
 625
 626         list_for_each_entry_safe (mtt, tmp, &destroy_list, odp_destroy.elm)
 627                 free_implicit_child_mr(mtt, false);
 628
 629         mlx5_mr_cache_free(dev, imr);
 630         ib_umem_odp_release(odp_imr);
 631 }
 632
 633 /**
 634  * mlx5_ib_fence_odp_mr - Stop all access to the ODP MR
 635  * @mr: to fence
 636  *
 637  * On return no parallel threads will be touching this MR and no DMA will be
 638  * active.
 639  */
 640 void mlx5_ib_fence_odp_mr(struct mlx5_ib_mr *mr)
 641 {
 642         /* Prevent new page faults and prefetch requests from succeeding */
 643         xa_erase(&mr->dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key));
 644
 645         /* Wait for all running page-fault handlers to finish. */
 646         synchronize_srcu(&mr->dev->odp_srcu);
 647
 648         if (atomic_read(&mr->num_deferred_work)) {
 649                 flush_workqueue(system_unbound_wq);
 650                 WARN_ON(atomic_read(&mr->num_deferred_work));
 651         }
 652
 653         dma_fence_odp_mr(mr);
 654 }
 655
 656 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1)
 657 static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp,
 658                              u64 user_va, size_t bcnt, u32 *bytes_mapped,
 659                              u32 flags)
 660 {
 661         int page_shift, ret, np;
 662         bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE;
 663         unsigned long current_seq;
 664         u64 access_mask;
 665         u64 start_idx;
 666
 667         page_shift = odp->page_shift;
 668         start_idx = (user_va - ib_umem_start(odp)) >> page_shift;
 669         access_mask = ODP_READ_ALLOWED_BIT;
 670
 671         if (odp->umem.writable && !downgrade)
 672                 access_mask |= ODP_WRITE_ALLOWED_BIT;
 673
 674         current_seq = mmu_interval_read_begin(&odp->notifier);
 675
 676         np = ib_umem_odp_map_dma_pages(odp, user_va, bcnt, access_mask,
 677                                        current_seq);
 678         if (np < 0)
 679                 return np;
 680
 681         mutex_lock(&odp->umem_mutex);
 682         if (!mmu_interval_read_retry(&odp->notifier, current_seq)) {
 683                 /*
 684                  * No need to check whether the MTTs really belong to
 685                  * this MR, since ib_umem_odp_map_dma_pages already
 686                  * checks this.
 687                  */
 688                 ret = mlx5_ib_update_xlt(mr, start_idx, np,
 689                                          page_shift, MLX5_IB_UPD_XLT_ATOMIC);
 690         } else {
 691                 ret = -EAGAIN;
 692         }
 693         mutex_unlock(&odp->umem_mutex);
 694
 695         if (ret < 0) {
 696                 if (ret != -EAGAIN)
 697                         mlx5_ib_err(mr->dev,
 698                                     "Failed to update mkey page tables\n");
 699                 goto out;
 700         }
 701
 702         if (bytes_mapped) {
 703                 u32 new_mappings = (np << page_shift) -
 704                         (user_va - round_down(user_va, 1 << page_shift));
 705
 706                 *bytes_mapped += min_t(u32, new_mappings, bcnt);
 707         }
 708
 709         return np << (page_shift - PAGE_SHIFT);
 710
 711 out:
 712         return ret;
 713 }
 714
 715 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr,
 716                                  struct ib_umem_odp *odp_imr, u64 user_va,
 717                                  size_t bcnt, u32 *bytes_mapped, u32 flags)
 718 {
 719         unsigned long end_idx = (user_va + bcnt - 1) >> MLX5_IMR_MTT_SHIFT;
 720         unsigned long upd_start_idx = end_idx + 1;
 721         unsigned long upd_len = 0;
 722         unsigned long npages = 0;
 723         int err;
 724         int ret;
 725
 726         if (unlikely(user_va >= mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE ||
 727                      mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE - user_va < bcnt))
 728                 return -EFAULT;
 729
 730         /* Fault each child mr that intersects with our interval. */
 731         while (bcnt) {
 732                 unsigned long idx = user_va >> MLX5_IMR_MTT_SHIFT;
 733                 struct ib_umem_odp *umem_odp;
 734                 struct mlx5_ib_mr *mtt;
 735                 u64 len;
 736
 737                 mtt = xa_load(&imr->implicit_children, idx);
 738                 if (unlikely(!mtt)) {
 739                         mtt = implicit_get_child_mr(imr, idx);
 740                         if (IS_ERR(mtt)) {
 741                                 ret = PTR_ERR(mtt);
 742                                 goto out;
 743                         }
 744                         upd_start_idx = min(upd_start_idx, idx);
 745                         upd_len = idx - upd_start_idx + 1;
 746                 }
 747
 748                 umem_odp = to_ib_umem_odp(mtt->umem);
 749                 len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) -
 750                       user_va;
 751
 752                 ret = pagefault_real_mr(mtt, umem_odp, user_va, len,
 753                                         bytes_mapped, flags);
 754                 if (ret < 0)
 755                         goto out;
 756                 user_va += len;
 757                 bcnt -= len;
 758                 npages += ret;
 759         }
 760
 761         ret = npages;
 762
 763         /*
 764          * Any time the implicit_children are changed we must perform an
 765          * update of the xlt before exiting to ensure the HW and the
 766          * implicit_children remains synchronized.
 767          */
 768 out:
 769         if (likely(!upd_len))
 770                 return ret;
 771
 772         /*
 773          * Notice this is not strictly ordered right, the KSM is updated after
 774          * the implicit_children is updated, so a parallel page fault could
 775          * see a MR that is not yet visible in the KSM.  This is similar to a
 776          * parallel page fault seeing a MR that is being concurrently removed
 777          * from the KSM. Both of these improbable situations are resolved
 778          * safely by resuming the HW and then taking another page fault. The
 779          * next pagefault handler will see the new information.
 780          */
 781         mutex_lock(&odp_imr->umem_mutex);
 782         err = mlx5_ib_update_xlt(imr, upd_start_idx, upd_len, 0,
 783                                  MLX5_IB_UPD_XLT_INDIRECT |
 784                                          MLX5_IB_UPD_XLT_ATOMIC);
 785         mutex_unlock(&odp_imr->umem_mutex);
 786         if (err) {
 787                 mlx5_ib_err(imr->dev, "Failed to update PAS\n");
 788                 return err;
 789         }
 790         return ret;
 791 }
 792
 793 /*
 794  * Returns:
 795  *  -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are
 796  *           not accessible, or the MR is no longer valid.
 797  *  -EAGAIN/-ENOMEM: The operation should be retried
 798  *
 799  *  -EINVAL/others: General internal malfunction
 800  *  >0: Number of pages mapped
 801  */
 802 static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt,
 803                         u32 *bytes_mapped, u32 flags)
 804 {
 805         struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem);
 806
 807         if (unlikely(io_virt < mr->mmkey.iova))
 808                 return -EFAULT;
 809
 810         if (!odp->is_implicit_odp) {
 811                 u64 user_va;
 812
 813                 if (check_add_overflow(io_virt - mr->mmkey.iova,
 814                                        (u64)odp->umem.address, &user_va))
 815                         return -EFAULT;
 816                 if (unlikely(user_va >= ib_umem_end(odp) ||
 817                              ib_umem_end(odp) - user_va < bcnt))
 818                         return -EFAULT;
 819                 return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped,
 820                                          flags);
 821         }
 822         return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped,
 823                                      flags);
 824 }
 825
 826 struct pf_frame {
 827         struct pf_frame *next;
 828         u32 key;
 829         u64 io_virt;
 830         size_t bcnt;
 831         int depth;
 832 };
 833
 834 static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key)
 835 {
 836         if (!mmkey)
 837                 return false;
 838         if (mmkey->type == MLX5_MKEY_MW)
 839                 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key);
 840         return mmkey->key == key;
 841 }
 842
 843 static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey)
 844 {
 845         struct mlx5_ib_mw *mw;
 846         struct mlx5_ib_devx_mr *devx_mr;
 847
 848         if (mmkey->type == MLX5_MKEY_MW) {
 849                 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey);
 850                 return mw->ndescs;
 851         }
 852
 853         devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr,
 854                                mmkey);
 855         return devx_mr->ndescs;
 856 }
 857
 858 /*
 859  * Handle a single data segment in a page-fault WQE or RDMA region.
 860  *
 861  * Returns number of OS pages retrieved on success. The caller may continue to
 862  * the next data segment.
 863  * Can return the following error codes:
 864  * -EAGAIN to designate a temporary error. The caller will abort handling the
 865  *  page fault and resolve it.
 866  * -EFAULT when there's an error mapping the requested pages. The caller will
 867  *  abort the page fault handling.
 868  */
 869 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev,
 870                                          struct ib_pd *pd, u32 key,
 871                                          u64 io_virt, size_t bcnt,
 872                                          u32 *bytes_committed,
 873                                          u32 *bytes_mapped)
 874 {
 875         int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0;
 876         struct pf_frame *head = NULL, *frame;
 877         struct mlx5_core_mkey *mmkey;
 878         struct mlx5_ib_mr *mr;
 879         struct mlx5_klm *pklm;
 880         u32 *out = NULL;
 881         size_t offset;
 882         int ndescs;
 883
 884         srcu_key = srcu_read_lock(&dev->odp_srcu);
 885
 886         io_virt += *bytes_committed;
 887         bcnt -= *bytes_committed;
 888
 889 next_mr:
 890         mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key));
 891         if (!mmkey) {
 892                 mlx5_ib_dbg(
 893                         dev,
 894                         "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n",
 895                         key);
 896                 if (bytes_mapped)
 897                         *bytes_mapped += bcnt;
 898                 /*
 899                  * The user could specify a SGL with multiple lkeys and only
 900                  * some of them are ODP. Treat the non-ODP ones as fully
 901                  * faulted.
 902                  */
 903                 ret = 0;
 904                 goto srcu_unlock;
 905         }
 906         if (!mkey_is_eq(mmkey, key)) {
 907                 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key);
 908                 ret = -EFAULT;
 909                 goto srcu_unlock;
 910         }
 911
 912         switch (mmkey->type) {
 913         case MLX5_MKEY_MR:
 914                 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
 915
 916                 ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0);
 917                 if (ret < 0)
 918                         goto srcu_unlock;
 919
 920                 /*
 921                  * When prefetching a page, page fault is generated
 922                  * in order to bring the page to the main memory.
 923                  * In the current flow, page faults are being counted.
 924                  */
 925                 mlx5_update_odp_stats(mr, faults, ret);
 926
 927                 npages += ret;
 928                 ret = 0;
 929                 break;
 930
 931         case MLX5_MKEY_MW:
 932         case MLX5_MKEY_INDIRECT_DEVX:
 933                 ndescs = get_indirect_num_descs(mmkey);
 934
 935                 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) {
 936                         mlx5_ib_dbg(dev, "indirection level exceeded\n");
 937                         ret = -EFAULT;
 938                         goto srcu_unlock;
 939                 }
 940
 941                 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) +
 942                         sizeof(*pklm) * (ndescs - 2);
 943
 944                 if (outlen > cur_outlen) {
 945                         kfree(out);
 946                         out = kzalloc(outlen, GFP_KERNEL);
 947                         if (!out) {
 948                                 ret = -ENOMEM;
 949                                 goto srcu_unlock;
 950                         }
 951                         cur_outlen = outlen;
 952                 }
 953
 954                 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out,
 955                                                        bsf0_klm0_pas_mtt0_1);
 956
 957                 ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen);
 958                 if (ret)
 959                         goto srcu_unlock;
 960
 961                 offset = io_virt - MLX5_GET64(query_mkey_out, out,
 962                                               memory_key_mkey_entry.start_addr);
 963
 964                 for (i = 0; bcnt && i < ndescs; i++, pklm++) {
 965                         if (offset >= be32_to_cpu(pklm->bcount)) {
 966                                 offset -= be32_to_cpu(pklm->bcount);
 967                                 continue;
 968                         }
 969
 970                         frame = kzalloc(sizeof(*frame), GFP_KERNEL);
 971                         if (!frame) {
 972                                 ret = -ENOMEM;
 973                                 goto srcu_unlock;
 974                         }
 975
 976                         frame->key = be32_to_cpu(pklm->key);
 977                         frame->io_virt = be64_to_cpu(pklm->va) + offset;
 978                         frame->bcnt = min_t(size_t, bcnt,
 979                                             be32_to_cpu(pklm->bcount) - offset);
 980                         frame->depth = depth + 1;
 981                         frame->next = head;
 982                         head = frame;
 983
 984                         bcnt -= frame->bcnt;
 985                         offset = 0;
 986                 }
 987                 break;
 988
 989         default:
 990                 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type);
 991                 ret = -EFAULT;
 992                 goto srcu_unlock;
 993         }
 994
 995         if (head) {
 996                 frame = head;
 997                 head = frame->next;
 998
 999                 key = frame->key;
1000                 io_virt = frame->io_virt;
1001                 bcnt = frame->bcnt;
1002                 depth = frame->depth;
1003                 kfree(frame);
1004
1005                 goto next_mr;
1006         }
1007
1008 srcu_unlock:
1009         while (head) {
1010                 frame = head;
1011                 head = frame->next;
1012                 kfree(frame);
1013         }
1014         kfree(out);
1015
1016         srcu_read_unlock(&dev->odp_srcu, srcu_key);
1017         *bytes_committed = 0;
1018         return ret ? ret : npages;
1019 }
1020
1021 /**
1022  * Parse a series of data segments for page fault handling.
1023  *
1024  * @pfault contains page fault information.
1025  * @wqe points at the first data segment in the WQE.
1026  * @wqe_end points after the end of the WQE.
1027  * @bytes_mapped receives the number of bytes that the function was able to
1028  *               map. This allows the caller to decide intelligently whether
1029  *               enough memory was mapped to resolve the page fault
1030  *               successfully (e.g. enough for the next MTU, or the entire
1031  *               WQE).
1032  * @total_wqe_bytes receives the total data size of this WQE in bytes (minus
1033  *                  the committed bytes).
1034  *
1035  * Returns the number of pages loaded if positive, zero for an empty WQE, or a
1036  * negative error code.
1037  */
1038 static int pagefault_data_segments(struct mlx5_ib_dev *dev,
1039                                    struct mlx5_pagefault *pfault,
1040                                    void *wqe,
1041                                    void *wqe_end, u32 *bytes_mapped,
1042                                    u32 *total_wqe_bytes, bool receive_queue)
1043 {
1044         int ret = 0, npages = 0;
1045         u64 io_virt;
1046         u32 key;
1047         u32 byte_count;
1048         size_t bcnt;
1049         int inline_segment;
1050
1051         if (bytes_mapped)
1052                 *bytes_mapped = 0;
1053         if (total_wqe_bytes)
1054                 *total_wqe_bytes = 0;
1055
1056         while (wqe < wqe_end) {
1057                 struct mlx5_wqe_data_seg *dseg = wqe;
1058
1059                 io_virt = be64_to_cpu(dseg->addr);
1060                 key = be32_to_cpu(dseg->lkey);
1061                 byte_count = be32_to_cpu(dseg->byte_count);
1062                 inline_segment = !!(byte_count &  MLX5_INLINE_SEG);
1063                 bcnt           = byte_count & ~MLX5_INLINE_SEG;
1064
1065                 if (inline_segment) {
1066                         bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK;
1067                         wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt,
1068                                      16);
1069                 } else {
1070                         wqe += sizeof(*dseg);
1071                 }
1072
1073                 /* receive WQE end of sg list. */
1074                 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY &&
1075                     io_virt == 0)
1076                         break;
1077
1078                 if (!inline_segment && total_wqe_bytes) {
1079                         *total_wqe_bytes += bcnt - min_t(size_t, bcnt,
1080                                         pfault->bytes_committed);
1081                 }
1082
1083                 /* A zero length data segment designates a length of 2GB. */
1084                 if (bcnt == 0)
1085                         bcnt = 1U << 31;
1086
1087                 if (inline_segment || bcnt <= pfault->bytes_committed) {
1088                         pfault->bytes_committed -=
1089                                 min_t(size_t, bcnt,
1090                                       pfault->bytes_committed);
1091                         continue;
1092                 }
1093
1094                 ret = pagefault_single_data_segment(dev, NULL, key,
1095                                                     io_virt, bcnt,
1096                                                     &pfault->bytes_committed,
1097                                                     bytes_mapped);
1098                 if (ret < 0)
1099                         break;
1100                 npages += ret;
1101         }
1102
1103         return ret < 0 ? ret : npages;
1104 }
1105
1106 /*
1107  * Parse initiator WQE. Advances the wqe pointer to point at the
1108  * scatter-gather list, and set wqe_end to the end of the WQE.
1109  */
1110 static int mlx5_ib_mr_initiator_pfault_handler(
1111         struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault,
1112         struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length)
1113 {
1114         struct mlx5_wqe_ctrl_seg *ctrl = *wqe;
1115         u16 wqe_index = pfault->wqe.wqe_index;
1116         struct mlx5_base_av *av;
1117         unsigned ds, opcode;
1118         u32 qpn = qp->trans_qp.base.mqp.qpn;
1119
1120         ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK;
1121         if (ds * MLX5_WQE_DS_UNITS > wqe_length) {
1122                 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n",
1123                             ds, wqe_length);
1124                 return -EFAULT;
1125         }
1126
1127         if (ds == 0) {
1128                 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n",
1129                             wqe_index, qpn);
1130                 return -EFAULT;
1131         }
1132
1133         *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS;
1134         *wqe += sizeof(*ctrl);
1135
1136         opcode = be32_to_cpu(ctrl->opmod_idx_opcode) &
1137                  MLX5_WQE_CTRL_OPCODE_MASK;
1138
1139         if (qp->ibqp.qp_type == IB_QPT_XRC_INI)
1140                 *wqe += sizeof(struct mlx5_wqe_xrc_seg);
1141
1142         if (qp->ibqp.qp_type == IB_QPT_UD ||
1143             qp->qp_sub_type == MLX5_IB_QPT_DCI) {
1144                 av = *wqe;
1145                 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV))
1146                         *wqe += sizeof(struct mlx5_av);
1147                 else
1148                         *wqe += sizeof(struct mlx5_base_av);
1149         }
1150
1151         switch (opcode) {
1152         case MLX5_OPCODE_RDMA_WRITE:
1153         case MLX5_OPCODE_RDMA_WRITE_IMM:
1154         case MLX5_OPCODE_RDMA_READ:
1155                 *wqe += sizeof(struct mlx5_wqe_raddr_seg);
1156                 break;
1157         case MLX5_OPCODE_ATOMIC_CS:
1158         case MLX5_OPCODE_ATOMIC_FA:
1159                 *wqe += sizeof(struct mlx5_wqe_raddr_seg);
1160                 *wqe += sizeof(struct mlx5_wqe_atomic_seg);
1161                 break;
1162         }
1163
1164         return 0;
1165 }
1166
1167 /*
1168  * Parse responder WQE and set wqe_end to the end of the WQE.
1169  */
1170 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev,
1171                                                    struct mlx5_ib_srq *srq,
1172                                                    void **wqe, void **wqe_end,
1173                                                    int wqe_length)
1174 {
1175         int wqe_size = 1 << srq->msrq.wqe_shift;
1176
1177         if (wqe_size > wqe_length) {
1178                 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n");
1179                 return -EFAULT;
1180         }
1181
1182         *wqe_end = *wqe + wqe_size;
1183         *wqe += sizeof(struct mlx5_wqe_srq_next_seg);
1184
1185         return 0;
1186 }
1187
1188 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev,
1189                                                   struct mlx5_ib_qp *qp,
1190                                                   void *wqe, void **wqe_end,
1191                                                   int wqe_length)
1192 {
1193         struct mlx5_ib_wq *wq = &qp->rq;
1194         int wqe_size = 1 << wq->wqe_shift;
1195
1196         if (qp->wq_sig) {
1197                 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n");
1198                 return -EFAULT;
1199         }
1200
1201         if (wqe_size > wqe_length) {
1202                 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n");
1203                 return -EFAULT;
1204         }
1205
1206         *wqe_end = wqe + wqe_size;
1207
1208         return 0;
1209 }
1210
1211 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev,
1212                                                        u32 wq_num, int pf_type)
1213 {
1214         struct mlx5_core_rsc_common *common = NULL;
1215         struct mlx5_core_srq *srq;
1216
1217         switch (pf_type) {
1218         case MLX5_WQE_PF_TYPE_RMP:
1219                 srq = mlx5_cmd_get_srq(dev, wq_num);
1220                 if (srq)
1221                         common = &srq->common;
1222                 break;
1223         case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE:
1224         case MLX5_WQE_PF_TYPE_RESP:
1225         case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC:
1226                 common = mlx5_core_res_hold(dev->mdev, wq_num, MLX5_RES_QP);
1227                 break;
1228         default:
1229                 break;
1230         }
1231
1232         return common;
1233 }
1234
1235 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res)
1236 {
1237         struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res;
1238
1239         return to_mibqp(mqp);
1240 }
1241
1242 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res)
1243 {
1244         struct mlx5_core_srq *msrq =
1245                 container_of(res, struct mlx5_core_srq, common);
1246
1247         return to_mibsrq(msrq);
1248 }
1249
1250 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev,
1251                                           struct mlx5_pagefault *pfault)
1252 {
1253         bool sq = pfault->type & MLX5_PFAULT_REQUESTOR;
1254         u16 wqe_index = pfault->wqe.wqe_index;
1255         void *wqe, *wqe_start = NULL, *wqe_end = NULL;
1256         u32 bytes_mapped, total_wqe_bytes;
1257         struct mlx5_core_rsc_common *res;
1258         int resume_with_error = 1;
1259         struct mlx5_ib_qp *qp;
1260         size_t bytes_copied;
1261         int ret = 0;
1262
1263         res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type);
1264         if (!res) {
1265                 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num);
1266                 return;
1267         }
1268
1269         if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ &&
1270             res->res != MLX5_RES_XSRQ) {
1271                 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n",
1272                             pfault->type);
1273                 goto resolve_page_fault;
1274         }
1275
1276         wqe_start = (void *)__get_free_page(GFP_KERNEL);
1277         if (!wqe_start) {
1278                 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n");
1279                 goto resolve_page_fault;
1280         }
1281
1282         wqe = wqe_start;
1283         qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL;
1284         if (qp && sq) {
1285                 ret = mlx5_ib_read_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE,
1286                                           &bytes_copied);
1287                 if (ret)
1288                         goto read_user;
1289                 ret = mlx5_ib_mr_initiator_pfault_handler(
1290                         dev, pfault, qp, &wqe, &wqe_end, bytes_copied);
1291         } else if (qp && !sq) {
1292                 ret = mlx5_ib_read_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE,
1293                                           &bytes_copied);
1294                 if (ret)
1295                         goto read_user;
1296                 ret = mlx5_ib_mr_responder_pfault_handler_rq(
1297                         dev, qp, wqe, &wqe_end, bytes_copied);
1298         } else if (!qp) {
1299                 struct mlx5_ib_srq *srq = res_to_srq(res);
1300
1301                 ret = mlx5_ib_read_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE,
1302                                            &bytes_copied);
1303                 if (ret)
1304                         goto read_user;
1305                 ret = mlx5_ib_mr_responder_pfault_handler_srq(
1306                         dev, srq, &wqe, &wqe_end, bytes_copied);
1307         }
1308
1309         if (ret < 0 || wqe >= wqe_end)
1310                 goto resolve_page_fault;
1311
1312         ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped,
1313                                       &total_wqe_bytes, !sq);
1314         if (ret == -EAGAIN)
1315                 goto out;
1316
1317         if (ret < 0 || total_wqe_bytes > bytes_mapped)
1318                 goto resolve_page_fault;
1319
1320 out:
1321         ret = 0;
1322         resume_with_error = 0;
1323
1324 read_user:
1325         if (ret)
1326                 mlx5_ib_err(
1327                         dev,
1328                         "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n",
1329                         ret, wqe_index, pfault->token);
1330
1331 resolve_page_fault:
1332         mlx5_ib_page_fault_resume(dev, pfault, resume_with_error);
1333         mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n",
1334                     pfault->wqe.wq_num, resume_with_error,
1335                     pfault->type);
1336         mlx5_core_res_put(res);
1337         free_page((unsigned long)wqe_start);
1338 }
1339
1340 static int pages_in_range(u64 address, u32 length)
1341 {
1342         return (ALIGN(address + length, PAGE_SIZE) -
1343                 (address & PAGE_MASK)) >> PAGE_SHIFT;
1344 }
1345
1346 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev,
1347                                            struct mlx5_pagefault *pfault)
1348 {
1349         u64 address;
1350         u32 length;
1351         u32 prefetch_len = pfault->bytes_committed;
1352         int prefetch_activated = 0;
1353         u32 rkey = pfault->rdma.r_key;
1354         int ret;
1355
1356         /* The RDMA responder handler handles the page fault in two parts.
1357          * First it brings the necessary pages for the current packet
1358          * (and uses the pfault context), and then (after resuming the QP)
1359          * prefetches more pages. The second operation cannot use the pfault
1360          * context and therefore uses the dummy_pfault context allocated on
1361          * the stack */
1362         pfault->rdma.rdma_va += pfault->bytes_committed;
1363         pfault->rdma.rdma_op_len -= min(pfault->bytes_committed,
1364                                          pfault->rdma.rdma_op_len);
1365         pfault->bytes_committed = 0;
1366
1367         address = pfault->rdma.rdma_va;
1368         length  = pfault->rdma.rdma_op_len;
1369
1370         /* For some operations, the hardware cannot tell the exact message
1371          * length, and in those cases it reports zero. Use prefetch
1372          * logic. */
1373         if (length == 0) {
1374                 prefetch_activated = 1;
1375                 length = pfault->rdma.packet_size;
1376                 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len);
1377         }
1378
1379         ret = pagefault_single_data_segment(dev, NULL, rkey, address, length,
1380                                             &pfault->bytes_committed, NULL);
1381         if (ret == -EAGAIN) {
1382                 /* We're racing with an invalidation, don't prefetch */
1383                 prefetch_activated = 0;
1384         } else if (ret < 0 || pages_in_range(address, length) > ret) {
1385                 mlx5_ib_page_fault_resume(dev, pfault, 1);
1386                 if (ret != -ENOENT)
1387                         mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n",
1388                                     ret, pfault->token, pfault->type);
1389                 return;
1390         }
1391
1392         mlx5_ib_page_fault_resume(dev, pfault, 0);
1393         mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n",
1394                     pfault->token, pfault->type,
1395                     prefetch_activated);
1396
1397         /* At this point, there might be a new pagefault already arriving in
1398          * the eq, switch to the dummy pagefault for the rest of the
1399          * processing. We're still OK with the objects being alive as the
1400          * work-queue is being fenced. */
1401
1402         if (prefetch_activated) {
1403                 u32 bytes_committed = 0;
1404
1405                 ret = pagefault_single_data_segment(dev, NULL, rkey, address,
1406                                                     prefetch_len,
1407                                                     &bytes_committed, NULL);
1408                 if (ret < 0 && ret != -EAGAIN) {
1409                         mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n",
1410                                     ret, pfault->token, address, prefetch_len);
1411                 }
1412         }
1413 }
1414
1415 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault)
1416 {
1417         u8 event_subtype = pfault->event_subtype;
1418
1419         switch (event_subtype) {
1420         case MLX5_PFAULT_SUBTYPE_WQE:
1421                 mlx5_ib_mr_wqe_pfault_handler(dev, pfault);
1422                 break;
1423         case MLX5_PFAULT_SUBTYPE_RDMA:
1424                 mlx5_ib_mr_rdma_pfault_handler(dev, pfault);
1425                 break;
1426         default:
1427                 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n",
1428                             event_subtype);
1429                 mlx5_ib_page_fault_resume(dev, pfault, 1);
1430         }
1431 }
1432
1433 static void mlx5_ib_eqe_pf_action(struct work_struct *work)
1434 {
1435         struct mlx5_pagefault *pfault = container_of(work,
1436                                                      struct mlx5_pagefault,
1437                                                      work);
1438         struct mlx5_ib_pf_eq *eq = pfault->eq;
1439
1440         mlx5_ib_pfault(eq->dev, pfault);
1441         mempool_free(pfault, eq->pool);
1442 }
1443
1444 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq)
1445 {
1446         struct mlx5_eqe_page_fault *pf_eqe;
1447         struct mlx5_pagefault *pfault;
1448         struct mlx5_eqe *eqe;
1449         int cc = 0;
1450
1451         while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) {
1452                 pfault = mempool_alloc(eq->pool, GFP_ATOMIC);
1453                 if (!pfault) {
1454                         schedule_work(&eq->work);
1455                         break;
1456                 }
1457
1458                 pf_eqe = &eqe->data.page_fault;
1459                 pfault->event_subtype = eqe->sub_type;
1460                 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed);
1461
1462                 mlx5_ib_dbg(eq->dev,
1463                             "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n",
1464                             eqe->sub_type, pfault->bytes_committed);
1465
1466                 switch (eqe->sub_type) {
1467                 case MLX5_PFAULT_SUBTYPE_RDMA:
1468                         /* RDMA based event */
1469                         pfault->type =
1470                                 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24;
1471                         pfault->token =
1472                                 be32_to_cpu(pf_eqe->rdma.pftype_token) &
1473                                 MLX5_24BIT_MASK;
1474                         pfault->rdma.r_key =
1475                                 be32_to_cpu(pf_eqe->rdma.r_key);
1476                         pfault->rdma.packet_size =
1477                                 be16_to_cpu(pf_eqe->rdma.packet_length);
1478                         pfault->rdma.rdma_op_len =
1479                                 be32_to_cpu(pf_eqe->rdma.rdma_op_len);
1480                         pfault->rdma.rdma_va =
1481                                 be64_to_cpu(pf_eqe->rdma.rdma_va);
1482                         mlx5_ib_dbg(eq->dev,
1483                                     "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n",
1484                                     pfault->type, pfault->token,
1485                                     pfault->rdma.r_key);
1486                         mlx5_ib_dbg(eq->dev,
1487                                     "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n",
1488                                     pfault->rdma.rdma_op_len,
1489                                     pfault->rdma.rdma_va);
1490                         break;
1491
1492                 case MLX5_PFAULT_SUBTYPE_WQE:
1493                         /* WQE based event */
1494                         pfault->type =
1495                                 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7;
1496                         pfault->token =
1497                                 be32_to_cpu(pf_eqe->wqe.token);
1498                         pfault->wqe.wq_num =
1499                                 be32_to_cpu(pf_eqe->wqe.pftype_wq) &
1500                                 MLX5_24BIT_MASK;
1501                         pfault->wqe.wqe_index =
1502                                 be16_to_cpu(pf_eqe->wqe.wqe_index);
1503                         pfault->wqe.packet_size =
1504                                 be16_to_cpu(pf_eqe->wqe.packet_length);
1505                         mlx5_ib_dbg(eq->dev,
1506                                     "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n",
1507                                     pfault->type, pfault->token,
1508                                     pfault->wqe.wq_num,
1509                                     pfault->wqe.wqe_index);
1510                         break;
1511
1512                 default:
1513                         mlx5_ib_warn(eq->dev,
1514                                      "Unsupported page fault event sub-type: 0x%02hhx\n",
1515                                      eqe->sub_type);
1516                         /* Unsupported page faults should still be
1517                          * resolved by the page fault handler
1518                          */
1519                 }
1520
1521                 pfault->eq = eq;
1522                 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action);
1523                 queue_work(eq->wq, &pfault->work);
1524
1525                 cc = mlx5_eq_update_cc(eq->core, ++cc);
1526         }
1527
1528         mlx5_eq_update_ci(eq->core, cc, 1);
1529 }
1530
1531 static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type,
1532                              void *data)
1533 {
1534         struct mlx5_ib_pf_eq *eq =
1535                 container_of(nb, struct mlx5_ib_pf_eq, irq_nb);
1536         unsigned long flags;
1537
1538         if (spin_trylock_irqsave(&eq->lock, flags)) {
1539                 mlx5_ib_eq_pf_process(eq);
1540                 spin_unlock_irqrestore(&eq->lock, flags);
1541         } else {
1542                 schedule_work(&eq->work);
1543         }
1544
1545         return IRQ_HANDLED;
1546 }
1547
1548 /* mempool_refill() was proposed but unfortunately wasn't accepted
1549  * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html
1550  * Cheap workaround.
1551  */
1552 static void mempool_refill(mempool_t *pool)
1553 {
1554         while (pool->curr_nr < pool->min_nr)
1555                 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool);
1556 }
1557
1558 static void mlx5_ib_eq_pf_action(struct work_struct *work)
1559 {
1560         struct mlx5_ib_pf_eq *eq =
1561                 container_of(work, struct mlx5_ib_pf_eq, work);
1562
1563         mempool_refill(eq->pool);
1564
1565         spin_lock_irq(&eq->lock);
1566         mlx5_ib_eq_pf_process(eq);
1567         spin_unlock_irq(&eq->lock);
1568 }
1569
1570 enum {
1571         MLX5_IB_NUM_PF_EQE      = 0x1000,
1572         MLX5_IB_NUM_PF_DRAIN    = 64,
1573 };
1574
1575 static int
1576 mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
1577 {
1578         struct mlx5_eq_param param = {};
1579         int err;
1580
1581         INIT_WORK(&eq->work, mlx5_ib_eq_pf_action);
1582         spin_lock_init(&eq->lock);
1583         eq->dev = dev;
1584
1585         eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN,
1586                                                sizeof(struct mlx5_pagefault));
1587         if (!eq->pool)
1588                 return -ENOMEM;
1589
1590         eq->wq = alloc_workqueue("mlx5_ib_page_fault",
1591                                  WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM,
1592                                  MLX5_NUM_CMD_EQE);
1593         if (!eq->wq) {
1594                 err = -ENOMEM;
1595                 goto err_mempool;
1596         }
1597
1598         eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int;
1599         param = (struct mlx5_eq_param) {
1600                 .irq_index = 0,
1601                 .nent = MLX5_IB_NUM_PF_EQE,
1602         };
1603         param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT;
1604         eq->core = mlx5_eq_create_generic(dev->mdev, &param);
1605         if (IS_ERR(eq->core)) {
1606                 err = PTR_ERR(eq->core);
1607                 goto err_wq;
1608         }
1609         err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb);
1610         if (err) {
1611                 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err);
1612                 goto err_eq;
1613         }
1614
1615         return 0;
1616 err_eq:
1617         mlx5_eq_destroy_generic(dev->mdev, eq->core);
1618 err_wq:
1619         destroy_workqueue(eq->wq);
1620 err_mempool:
1621         mempool_destroy(eq->pool);
1622         return err;
1623 }
1624
1625 static int
1626 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq)
1627 {
1628         int err;
1629
1630         mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb);
1631         err = mlx5_eq_destroy_generic(dev->mdev, eq->core);
1632         cancel_work_sync(&eq->work);
1633         destroy_workqueue(eq->wq);
1634         mempool_destroy(eq->pool);
1635
1636         return err;
1637 }
1638
1639 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent)
1640 {
1641         if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT))
1642                 return;
1643
1644         switch (ent->order - 2) {
1645         case MLX5_IMR_MTT_CACHE_ENTRY:
1646                 ent->page = PAGE_SHIFT;
1647                 ent->xlt = MLX5_IMR_MTT_ENTRIES *
1648                            sizeof(struct mlx5_mtt) /
1649                            MLX5_IB_UMR_OCTOWORD;
1650                 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT;
1651                 ent->limit = 0;
1652                 break;
1653
1654         case MLX5_IMR_KSM_CACHE_ENTRY:
1655                 ent->page = MLX5_KSM_PAGE_SHIFT;
1656                 ent->xlt = mlx5_imr_ksm_entries *
1657                            sizeof(struct mlx5_klm) /
1658                            MLX5_IB_UMR_OCTOWORD;
1659                 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM;
1660                 ent->limit = 0;
1661                 break;
1662         }
1663 }
1664
1665 static const struct ib_device_ops mlx5_ib_dev_odp_ops = {
1666         .advise_mr = mlx5_ib_advise_mr,
1667 };
1668
1669 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev)
1670 {
1671         int ret = 0;
1672
1673         if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT))
1674                 return ret;
1675
1676         ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops);
1677
1678         if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) {
1679                 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey);
1680                 if (ret) {
1681                         mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret);
1682                         return ret;
1683                 }
1684         }
1685
1686         ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq);
1687
1688         return ret;
1689 }
1690
1691 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev)
1692 {
1693         if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT))
1694                 return;
1695
1696         mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq);
1697 }
1698
1699 int mlx5_ib_odp_init(void)
1700 {
1701         mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) -
1702                                        MLX5_IMR_MTT_BITS);
1703
1704         return 0;
1705 }
1706
1707 struct prefetch_mr_work {
1708         struct work_struct work;
1709         u32 pf_flags;
1710         u32 num_sge;
1711         struct {
1712                 u64 io_virt;
1713                 struct mlx5_ib_mr *mr;
1714                 size_t length;
1715         } frags[];
1716 };
1717
1718 static void destroy_prefetch_work(struct prefetch_mr_work *work)
1719 {
1720         u32 i;
1721
1722         for (i = 0; i < work->num_sge; ++i)
1723                 atomic_dec(&work->frags[i].mr->num_deferred_work);
1724         kvfree(work);
1725 }
1726
1727 static struct mlx5_ib_mr *
1728 get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice,
1729                     u32 lkey)
1730 {
1731         struct mlx5_ib_dev *dev = to_mdev(pd->device);
1732         struct mlx5_core_mkey *mmkey;
1733         struct ib_umem_odp *odp;
1734         struct mlx5_ib_mr *mr;
1735
1736         lockdep_assert_held(&dev->odp_srcu);
1737
1738         mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey));
1739         if (!mmkey || mmkey->key != lkey || mmkey->type != MLX5_MKEY_MR)
1740                 return NULL;
1741
1742         mr = container_of(mmkey, struct mlx5_ib_mr, mmkey);
1743
1744         if (mr->ibmr.pd != pd)
1745                 return NULL;
1746
1747         odp = to_ib_umem_odp(mr->umem);
1748
1749         /* prefetch with write-access must be supported by the MR */
1750         if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE &&
1751             !odp->umem.writable)
1752                 return NULL;
1753
1754         return mr;
1755 }
1756
1757 static void mlx5_ib_prefetch_mr_work(struct work_struct *w)
1758 {
1759         struct prefetch_mr_work *work =
1760                 container_of(w, struct prefetch_mr_work, work);
1761         u32 bytes_mapped = 0;
1762         u32 i;
1763
1764         for (i = 0; i < work->num_sge; ++i)
1765                 pagefault_mr(work->frags[i].mr, work->frags[i].io_virt,
1766                              work->frags[i].length, &bytes_mapped,
1767                              work->pf_flags);
1768
1769         destroy_prefetch_work(work);
1770 }
1771
1772 static bool init_prefetch_work(struct ib_pd *pd,
1773                                enum ib_uverbs_advise_mr_advice advice,
1774                                u32 pf_flags, struct prefetch_mr_work *work,
1775                                struct ib_sge *sg_list, u32 num_sge)
1776 {
1777         u32 i;
1778
1779         INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work);
1780         work->pf_flags = pf_flags;
1781
1782         for (i = 0; i < num_sge; ++i) {
1783                 work->frags[i].io_virt = sg_list[i].addr;
1784                 work->frags[i].length = sg_list[i].length;
1785                 work->frags[i].mr =
1786                         get_prefetchable_mr(pd, advice, sg_list[i].lkey);
1787                 if (!work->frags[i].mr) {
1788                         work->num_sge = i - 1;
1789                         if (i)
1790                                 destroy_prefetch_work(work);
1791                         return false;
1792                 }
1793
1794                 /* Keep the MR pointer will valid outside the SRCU */
1795                 atomic_inc(&work->frags[i].mr->num_deferred_work);
1796         }
1797         work->num_sge = num_sge;
1798         return true;
1799 }
1800
1801 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd,
1802                                     enum ib_uverbs_advise_mr_advice advice,
1803                                     u32 pf_flags, struct ib_sge *sg_list,
1804                                     u32 num_sge)
1805 {
1806         struct mlx5_ib_dev *dev = to_mdev(pd->device);
1807         u32 bytes_mapped = 0;
1808         int srcu_key;
1809         int ret = 0;
1810         u32 i;
1811
1812         srcu_key = srcu_read_lock(&dev->odp_srcu);
1813         for (i = 0; i < num_sge; ++i) {
1814                 struct mlx5_ib_mr *mr;
1815
1816                 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey);
1817                 if (!mr) {
1818                         ret = -ENOENT;
1819                         goto out;
1820                 }
1821                 ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length,
1822                                    &bytes_mapped, pf_flags);
1823                 if (ret < 0)
1824                         goto out;
1825         }
1826         ret = 0;
1827
1828 out:
1829         srcu_read_unlock(&dev->odp_srcu, srcu_key);
1830         return ret;
1831 }
1832
1833 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd,
1834                                enum ib_uverbs_advise_mr_advice advice,
1835                                u32 flags, struct ib_sge *sg_list, u32 num_sge)
1836 {
1837         struct mlx5_ib_dev *dev = to_mdev(pd->device);
1838         u32 pf_flags = 0;
1839         struct prefetch_mr_work *work;
1840         int srcu_key;
1841
1842         if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH)
1843                 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE;
1844
1845         if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH)
1846                 return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list,
1847                                                 num_sge);
1848
1849         work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL);
1850         if (!work)
1851                 return -ENOMEM;
1852
1853         srcu_key = srcu_read_lock(&dev->odp_srcu);
1854         if (!init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge)) {
1855                 srcu_read_unlock(&dev->odp_srcu, srcu_key);
1856                 return -EINVAL;
1857         }
1858         queue_work(system_unbound_wq, &work->work);
1859         srcu_read_unlock(&dev->odp_srcu, srcu_key);
1860         return 0;
1861 }