drivers/infiniband/hw/mlx5/devx.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2018, Mellanox Technologies inc.  All rights reserved.
   4  */
   5
   6 #include <rdma/ib_user_verbs.h>
   7 #include <rdma/ib_verbs.h>
   8 #include <rdma/uverbs_types.h>
   9 #include <rdma/uverbs_ioctl.h>
  10 #include <rdma/mlx5_user_ioctl_cmds.h>
  11 #include <rdma/ib_umem.h>
  12 #include <linux/mlx5/driver.h>
  13 #include <linux/mlx5/fs.h>
  14 #include "mlx5_ib.h"
  15
  16 #define UVERBS_MODULE_NAME mlx5_ib
  17 #include <rdma/uverbs_named_ioctl.h>
  18
  19 #define MLX5_MAX_DESTROY_INBOX_SIZE_DW MLX5_ST_SZ_DW(delete_fte_in)
  20 struct devx_obj {
  21         struct mlx5_core_dev    *mdev;
  22         u64                     obj_id;
  23         u32                     dinlen; /* destroy inbox length */
  24         u32                     dinbox[MLX5_MAX_DESTROY_INBOX_SIZE_DW];
  25 };
  26
  27 struct devx_umem {
  28         struct mlx5_core_dev            *mdev;
  29         struct ib_umem                  *umem;
  30         u32                             page_offset;
  31         int                             page_shift;
  32         int                             ncont;
  33         u32                             dinlen;
  34         u32                             dinbox[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)];
  35 };
  36
  37 struct devx_umem_reg_cmd {
  38         void                            *in;
  39         u32                             inlen;
  40         u32                             out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
  41 };
  42
  43 static struct mlx5_ib_ucontext *devx_ufile2uctx(struct ib_uverbs_file *file)
  44 {
  45         return to_mucontext(ib_uverbs_get_ucontext(file));
  46 }
  47
  48 int mlx5_ib_devx_create(struct mlx5_ib_dev *dev)
  49 {
  50         u32 in[MLX5_ST_SZ_DW(create_uctx_in)] = {0};
  51         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
  52         u64 general_obj_types;
  53         void *hdr;
  54         int err;
  55         u16 uid;
  56
  57         hdr = MLX5_ADDR_OF(create_uctx_in, in, hdr);
  58
  59         general_obj_types = MLX5_CAP_GEN_64(dev->mdev, general_obj_types);
  60         if (!(general_obj_types & MLX5_GENERAL_OBJ_TYPES_CAP_UCTX) ||
  61             !(general_obj_types & MLX5_GENERAL_OBJ_TYPES_CAP_UMEM))
  62                 return -EINVAL;
  63
  64         MLX5_SET(general_obj_in_cmd_hdr, hdr, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
  65         MLX5_SET(general_obj_in_cmd_hdr, hdr, obj_type, MLX5_OBJ_TYPE_UCTX);
  66
  67         err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
  68         if (err)
  69                 return err;
  70
  71         uid = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
  72         return uid;
  73 }
  74
  75 void mlx5_ib_devx_destroy(struct mlx5_ib_dev *dev, u16 uid)
  76 {
  77         u32 in[MLX5_ST_SZ_DW(general_obj_in_cmd_hdr)] = {0};
  78         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)] = {0};
  79
  80         MLX5_SET(general_obj_in_cmd_hdr, in, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
  81         MLX5_SET(general_obj_in_cmd_hdr, in, obj_type, MLX5_OBJ_TYPE_UCTX);
  82         MLX5_SET(general_obj_in_cmd_hdr, in, obj_id, uid);
  83
  84         mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out));
  85 }
  86
  87 bool mlx5_ib_devx_is_flow_dest(void *obj, int *dest_id, int *dest_type)
  88 {
  89         struct devx_obj *devx_obj = obj;
  90         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox, opcode);
  91
  92         switch (opcode) {
  93         case MLX5_CMD_OP_DESTROY_TIR:
  94                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_TIR;
  95                 *dest_id = MLX5_GET(general_obj_in_cmd_hdr, devx_obj->dinbox,
  96                                     obj_id);
  97                 return true;
  98
  99         case MLX5_CMD_OP_DESTROY_FLOW_TABLE:
 100                 *dest_type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE;
 101                 *dest_id = MLX5_GET(destroy_flow_table_in, devx_obj->dinbox,
 102                                     table_id);
 103                 return true;
 104         default:
 105                 return false;
 106         }
 107 }
 108
 109 /*
 110  * As the obj_id in the firmware is not globally unique the object type
 111  * must be considered upon checking for a valid object id.
 112  * For that the opcode of the creator command is encoded as part of the obj_id.
 113  */
 114 static u64 get_enc_obj_id(u16 opcode, u32 obj_id)
 115 {
 116         return ((u64)opcode << 32) | obj_id;
 117 }
 118
 119 static int devx_is_valid_obj_id(struct devx_obj *obj, const void *in)
 120 {
 121         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 122         u64 obj_id;
 123
 124         switch (opcode) {
 125         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 126         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 127                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_GENERAL_OBJECT,
 128                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 129                                                  obj_id));
 130                 break;
 131         case MLX5_CMD_OP_QUERY_MKEY:
 132                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_MKEY,
 133                                         MLX5_GET(query_mkey_in, in,
 134                                                  mkey_index));
 135                 break;
 136         case MLX5_CMD_OP_QUERY_CQ:
 137                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 138                                         MLX5_GET(query_cq_in, in, cqn));
 139                 break;
 140         case MLX5_CMD_OP_MODIFY_CQ:
 141                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_CQ,
 142                                         MLX5_GET(modify_cq_in, in, cqn));
 143                 break;
 144         case MLX5_CMD_OP_QUERY_SQ:
 145                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 146                                         MLX5_GET(query_sq_in, in, sqn));
 147                 break;
 148         case MLX5_CMD_OP_MODIFY_SQ:
 149                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SQ,
 150                                         MLX5_GET(modify_sq_in, in, sqn));
 151                 break;
 152         case MLX5_CMD_OP_QUERY_RQ:
 153                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 154                                         MLX5_GET(query_rq_in, in, rqn));
 155                 break;
 156         case MLX5_CMD_OP_MODIFY_RQ:
 157                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 158                                         MLX5_GET(modify_rq_in, in, rqn));
 159                 break;
 160         case MLX5_CMD_OP_QUERY_RMP:
 161                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 162                                         MLX5_GET(query_rmp_in, in, rmpn));
 163                 break;
 164         case MLX5_CMD_OP_MODIFY_RMP:
 165                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RMP,
 166                                         MLX5_GET(modify_rmp_in, in, rmpn));
 167                 break;
 168         case MLX5_CMD_OP_QUERY_RQT:
 169                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 170                                         MLX5_GET(query_rqt_in, in, rqtn));
 171                 break;
 172         case MLX5_CMD_OP_MODIFY_RQT:
 173                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQT,
 174                                         MLX5_GET(modify_rqt_in, in, rqtn));
 175                 break;
 176         case MLX5_CMD_OP_QUERY_TIR:
 177                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 178                                         MLX5_GET(query_tir_in, in, tirn));
 179                 break;
 180         case MLX5_CMD_OP_MODIFY_TIR:
 181                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIR,
 182                                         MLX5_GET(modify_tir_in, in, tirn));
 183                 break;
 184         case MLX5_CMD_OP_QUERY_TIS:
 185                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 186                                         MLX5_GET(query_tis_in, in, tisn));
 187                 break;
 188         case MLX5_CMD_OP_MODIFY_TIS:
 189                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_TIS,
 190                                         MLX5_GET(modify_tis_in, in, tisn));
 191                 break;
 192         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 193                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 194                                         MLX5_GET(query_flow_table_in, in,
 195                                                  table_id));
 196                 break;
 197         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 198                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_TABLE,
 199                                         MLX5_GET(modify_flow_table_in, in,
 200                                                  table_id));
 201                 break;
 202         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 203                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_FLOW_GROUP,
 204                                         MLX5_GET(query_flow_group_in, in,
 205                                                  group_id));
 206                 break;
 207         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 208                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 209                                         MLX5_GET(query_fte_in, in,
 210                                                  flow_index));
 211                 break;
 212         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 213                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY,
 214                                         MLX5_GET(set_fte_in, in, flow_index));
 215                 break;
 216         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 217                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_Q_COUNTER,
 218                                         MLX5_GET(query_q_counter_in, in,
 219                                                  counter_set_id));
 220                 break;
 221         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 222                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_FLOW_COUNTER,
 223                                         MLX5_GET(query_flow_counter_in, in,
 224                                                  flow_counter_id));
 225                 break;
 226         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 227                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT,
 228                                         MLX5_GET(general_obj_in_cmd_hdr, in,
 229                                                  obj_id));
 230                 break;
 231         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 232                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 233                                         MLX5_GET(query_scheduling_element_in,
 234                                                  in, scheduling_element_id));
 235                 break;
 236         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 237                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT,
 238                                         MLX5_GET(modify_scheduling_element_in,
 239                                                  in, scheduling_element_id));
 240                 break;
 241         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 242                 obj_id = get_enc_obj_id(MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT,
 243                                         MLX5_GET(add_vxlan_udp_dport_in, in,
 244                                                  vxlan_udp_port));
 245                 break;
 246         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 247                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 248                                         MLX5_GET(query_l2_table_entry_in, in,
 249                                                  table_index));
 250                 break;
 251         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 252                 obj_id = get_enc_obj_id(MLX5_CMD_OP_SET_L2_TABLE_ENTRY,
 253                                         MLX5_GET(set_l2_table_entry_in, in,
 254                                                  table_index));
 255                 break;
 256         case MLX5_CMD_OP_QUERY_QP:
 257                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 258                                         MLX5_GET(query_qp_in, in, qpn));
 259                 break;
 260         case MLX5_CMD_OP_RST2INIT_QP:
 261                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 262                                         MLX5_GET(rst2init_qp_in, in, qpn));
 263                 break;
 264         case MLX5_CMD_OP_INIT2RTR_QP:
 265                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 266                                         MLX5_GET(init2rtr_qp_in, in, qpn));
 267                 break;
 268         case MLX5_CMD_OP_RTR2RTS_QP:
 269                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 270                                         MLX5_GET(rtr2rts_qp_in, in, qpn));
 271                 break;
 272         case MLX5_CMD_OP_RTS2RTS_QP:
 273                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 274                                         MLX5_GET(rts2rts_qp_in, in, qpn));
 275                 break;
 276         case MLX5_CMD_OP_SQERR2RTS_QP:
 277                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 278                                         MLX5_GET(sqerr2rts_qp_in, in, qpn));
 279                 break;
 280         case MLX5_CMD_OP_2ERR_QP:
 281                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 282                                         MLX5_GET(qp_2err_in, in, qpn));
 283                 break;
 284         case MLX5_CMD_OP_2RST_QP:
 285                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_QP,
 286                                         MLX5_GET(qp_2rst_in, in, qpn));
 287                 break;
 288         case MLX5_CMD_OP_QUERY_DCT:
 289                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 290                                         MLX5_GET(query_dct_in, in, dctn));
 291                 break;
 292         case MLX5_CMD_OP_QUERY_XRQ:
 293                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 294                                         MLX5_GET(query_xrq_in, in, xrqn));
 295                 break;
 296         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 297                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 298                                         MLX5_GET(query_xrc_srq_in, in,
 299                                                  xrc_srqn));
 300                 break;
 301         case MLX5_CMD_OP_ARM_XRC_SRQ:
 302                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRC_SRQ,
 303                                         MLX5_GET(arm_xrc_srq_in, in, xrc_srqn));
 304                 break;
 305         case MLX5_CMD_OP_QUERY_SRQ:
 306                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_SRQ,
 307                                         MLX5_GET(query_srq_in, in, srqn));
 308                 break;
 309         case MLX5_CMD_OP_ARM_RQ:
 310                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_RQ,
 311                                         MLX5_GET(arm_rq_in, in, srq_number));
 312                 break;
 313         case MLX5_CMD_OP_DRAIN_DCT:
 314         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 315                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_DCT,
 316                                         MLX5_GET(drain_dct_in, in, dctn));
 317                 break;
 318         case MLX5_CMD_OP_ARM_XRQ:
 319                 obj_id = get_enc_obj_id(MLX5_CMD_OP_CREATE_XRQ,
 320                                         MLX5_GET(arm_xrq_in, in, xrqn));
 321                 break;
 322         default:
 323                 return false;
 324         }
 325
 326         if (obj_id == obj->obj_id)
 327                 return true;
 328
 329         return false;
 330 }
 331
 332 static void devx_set_umem_valid(const void *in)
 333 {
 334         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 335
 336         switch (opcode) {
 337         case MLX5_CMD_OP_CREATE_MKEY:
 338                 MLX5_SET(create_mkey_in, in, mkey_umem_valid, 1);
 339                 break;
 340         case MLX5_CMD_OP_CREATE_CQ:
 341         {
 342                 void *cqc;
 343
 344                 MLX5_SET(create_cq_in, in, cq_umem_valid, 1);
 345                 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context);
 346                 MLX5_SET(cqc, cqc, dbr_umem_valid, 1);
 347                 break;
 348         }
 349         case MLX5_CMD_OP_CREATE_QP:
 350         {
 351                 void *qpc;
 352
 353                 qpc = MLX5_ADDR_OF(create_qp_in, in, qpc);
 354                 MLX5_SET(qpc, qpc, dbr_umem_valid, 1);
 355                 MLX5_SET(create_qp_in, in, wq_umem_valid, 1);
 356                 break;
 357         }
 358
 359         case MLX5_CMD_OP_CREATE_RQ:
 360         {
 361                 void *rqc, *wq;
 362
 363                 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx);
 364                 wq  = MLX5_ADDR_OF(rqc, rqc, wq);
 365                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 366                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 367                 break;
 368         }
 369
 370         case MLX5_CMD_OP_CREATE_SQ:
 371         {
 372                 void *sqc, *wq;
 373
 374                 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx);
 375                 wq = MLX5_ADDR_OF(sqc, sqc, wq);
 376                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 377                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 378                 break;
 379         }
 380
 381         case MLX5_CMD_OP_MODIFY_CQ:
 382                 MLX5_SET(modify_cq_in, in, cq_umem_valid, 1);
 383                 break;
 384
 385         case MLX5_CMD_OP_CREATE_RMP:
 386         {
 387                 void *rmpc, *wq;
 388
 389                 rmpc = MLX5_ADDR_OF(create_rmp_in, in, ctx);
 390                 wq = MLX5_ADDR_OF(rmpc, rmpc, wq);
 391                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 392                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 393                 break;
 394         }
 395
 396         case MLX5_CMD_OP_CREATE_XRQ:
 397         {
 398                 void *xrqc, *wq;
 399
 400                 xrqc = MLX5_ADDR_OF(create_xrq_in, in, xrq_context);
 401                 wq = MLX5_ADDR_OF(xrqc, xrqc, wq);
 402                 MLX5_SET(wq, wq, dbr_umem_valid, 1);
 403                 MLX5_SET(wq, wq, wq_umem_valid, 1);
 404                 break;
 405         }
 406
 407         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 408         {
 409                 void *xrc_srqc;
 410
 411                 MLX5_SET(create_xrc_srq_in, in, xrc_srq_umem_valid, 1);
 412                 xrc_srqc = MLX5_ADDR_OF(create_xrc_srq_in, in,
 413                                         xrc_srq_context_entry);
 414                 MLX5_SET(xrc_srqc, xrc_srqc, dbr_umem_valid, 1);
 415                 break;
 416         }
 417
 418         default:
 419                 return;
 420         }
 421 }
 422
 423 static bool devx_is_obj_create_cmd(const void *in, u16 *opcode)
 424 {
 425         *opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 426
 427         switch (*opcode) {
 428         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 429         case MLX5_CMD_OP_CREATE_MKEY:
 430         case MLX5_CMD_OP_CREATE_CQ:
 431         case MLX5_CMD_OP_ALLOC_PD:
 432         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 433         case MLX5_CMD_OP_CREATE_RMP:
 434         case MLX5_CMD_OP_CREATE_SQ:
 435         case MLX5_CMD_OP_CREATE_RQ:
 436         case MLX5_CMD_OP_CREATE_RQT:
 437         case MLX5_CMD_OP_CREATE_TIR:
 438         case MLX5_CMD_OP_CREATE_TIS:
 439         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 440         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 441         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 442         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 443         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 444         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 445         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 446         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 447         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 448         case MLX5_CMD_OP_CREATE_QP:
 449         case MLX5_CMD_OP_CREATE_SRQ:
 450         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 451         case MLX5_CMD_OP_CREATE_DCT:
 452         case MLX5_CMD_OP_CREATE_XRQ:
 453         case MLX5_CMD_OP_ATTACH_TO_MCG:
 454         case MLX5_CMD_OP_ALLOC_XRCD:
 455                 return true;
 456         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 457         {
 458                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 459                 if (op_mod == 0)
 460                         return true;
 461                 return false;
 462         }
 463         default:
 464                 return false;
 465         }
 466 }
 467
 468 static bool devx_is_obj_modify_cmd(const void *in)
 469 {
 470         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 471
 472         switch (opcode) {
 473         case MLX5_CMD_OP_MODIFY_GENERAL_OBJECT:
 474         case MLX5_CMD_OP_MODIFY_CQ:
 475         case MLX5_CMD_OP_MODIFY_RMP:
 476         case MLX5_CMD_OP_MODIFY_SQ:
 477         case MLX5_CMD_OP_MODIFY_RQ:
 478         case MLX5_CMD_OP_MODIFY_RQT:
 479         case MLX5_CMD_OP_MODIFY_TIR:
 480         case MLX5_CMD_OP_MODIFY_TIS:
 481         case MLX5_CMD_OP_MODIFY_FLOW_TABLE:
 482         case MLX5_CMD_OP_MODIFY_SCHEDULING_ELEMENT:
 483         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 484         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 485         case MLX5_CMD_OP_RST2INIT_QP:
 486         case MLX5_CMD_OP_INIT2RTR_QP:
 487         case MLX5_CMD_OP_RTR2RTS_QP:
 488         case MLX5_CMD_OP_RTS2RTS_QP:
 489         case MLX5_CMD_OP_SQERR2RTS_QP:
 490         case MLX5_CMD_OP_2ERR_QP:
 491         case MLX5_CMD_OP_2RST_QP:
 492         case MLX5_CMD_OP_ARM_XRC_SRQ:
 493         case MLX5_CMD_OP_ARM_RQ:
 494         case MLX5_CMD_OP_DRAIN_DCT:
 495         case MLX5_CMD_OP_ARM_DCT_FOR_KEY_VIOLATION:
 496         case MLX5_CMD_OP_ARM_XRQ:
 497                 return true;
 498         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 499         {
 500                 u16 op_mod = MLX5_GET(set_fte_in, in, op_mod);
 501
 502                 if (op_mod == 1)
 503                         return true;
 504                 return false;
 505         }
 506         default:
 507                 return false;
 508         }
 509 }
 510
 511 static bool devx_is_obj_query_cmd(const void *in)
 512 {
 513         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 514
 515         switch (opcode) {
 516         case MLX5_CMD_OP_QUERY_GENERAL_OBJECT:
 517         case MLX5_CMD_OP_QUERY_MKEY:
 518         case MLX5_CMD_OP_QUERY_CQ:
 519         case MLX5_CMD_OP_QUERY_RMP:
 520         case MLX5_CMD_OP_QUERY_SQ:
 521         case MLX5_CMD_OP_QUERY_RQ:
 522         case MLX5_CMD_OP_QUERY_RQT:
 523         case MLX5_CMD_OP_QUERY_TIR:
 524         case MLX5_CMD_OP_QUERY_TIS:
 525         case MLX5_CMD_OP_QUERY_Q_COUNTER:
 526         case MLX5_CMD_OP_QUERY_FLOW_TABLE:
 527         case MLX5_CMD_OP_QUERY_FLOW_GROUP:
 528         case MLX5_CMD_OP_QUERY_FLOW_TABLE_ENTRY:
 529         case MLX5_CMD_OP_QUERY_FLOW_COUNTER:
 530         case MLX5_CMD_OP_QUERY_MODIFY_HEADER_CONTEXT:
 531         case MLX5_CMD_OP_QUERY_SCHEDULING_ELEMENT:
 532         case MLX5_CMD_OP_QUERY_L2_TABLE_ENTRY:
 533         case MLX5_CMD_OP_QUERY_QP:
 534         case MLX5_CMD_OP_QUERY_SRQ:
 535         case MLX5_CMD_OP_QUERY_XRC_SRQ:
 536         case MLX5_CMD_OP_QUERY_DCT:
 537         case MLX5_CMD_OP_QUERY_XRQ:
 538                 return true;
 539         default:
 540                 return false;
 541         }
 542 }
 543
 544 static bool devx_is_whitelist_cmd(void *in)
 545 {
 546         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 547
 548         switch (opcode) {
 549         case MLX5_CMD_OP_QUERY_HCA_CAP:
 550         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 551                 return true;
 552         default:
 553                 return false;
 554         }
 555 }
 556
 557 static int devx_get_uid(struct mlx5_ib_ucontext *c, void *cmd_in)
 558 {
 559         if (devx_is_whitelist_cmd(cmd_in)) {
 560                 struct mlx5_ib_dev *dev;
 561
 562                 if (c->devx_uid)
 563                         return c->devx_uid;
 564
 565                 dev = to_mdev(c->ibucontext.device);
 566                 if (dev->devx_whitelist_uid)
 567                         return dev->devx_whitelist_uid;
 568
 569                 return -EOPNOTSUPP;
 570         }
 571
 572         if (!c->devx_uid)
 573                 return -EINVAL;
 574
 575         if (!capable(CAP_NET_RAW))
 576                 return -EPERM;
 577
 578         return c->devx_uid;
 579 }
 580 static bool devx_is_general_cmd(void *in)
 581 {
 582         u16 opcode = MLX5_GET(general_obj_in_cmd_hdr, in, opcode);
 583
 584         switch (opcode) {
 585         case MLX5_CMD_OP_QUERY_HCA_CAP:
 586         case MLX5_CMD_OP_QUERY_HCA_VPORT_CONTEXT:
 587         case MLX5_CMD_OP_QUERY_VPORT_STATE:
 588         case MLX5_CMD_OP_QUERY_ADAPTER:
 589         case MLX5_CMD_OP_QUERY_ISSI:
 590         case MLX5_CMD_OP_QUERY_NIC_VPORT_CONTEXT:
 591         case MLX5_CMD_OP_QUERY_ROCE_ADDRESS:
 592         case MLX5_CMD_OP_QUERY_VNIC_ENV:
 593         case MLX5_CMD_OP_QUERY_VPORT_COUNTER:
 594         case MLX5_CMD_OP_GET_DROPPED_PACKET_LOG:
 595         case MLX5_CMD_OP_NOP:
 596         case MLX5_CMD_OP_QUERY_CONG_STATUS:
 597         case MLX5_CMD_OP_QUERY_CONG_PARAMS:
 598         case MLX5_CMD_OP_QUERY_CONG_STATISTICS:
 599                 return true;
 600         default:
 601                 return false;
 602         }
 603 }
 604
 605 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_EQN)(
 606         struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
 607 {
 608         struct mlx5_ib_ucontext *c;
 609         struct mlx5_ib_dev *dev;
 610         int user_vector;
 611         int dev_eqn;
 612         unsigned int irqn;
 613         int err;
 614
 615         if (uverbs_copy_from(&user_vector, attrs,
 616                              MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC))
 617                 return -EFAULT;
 618
 619         c = devx_ufile2uctx(file);
 620         if (IS_ERR(c))
 621                 return PTR_ERR(c);
 622         dev = to_mdev(c->ibucontext.device);
 623
 624         err = mlx5_vector2eqn(dev->mdev, user_vector, &dev_eqn, &irqn);
 625         if (err < 0)
 626                 return err;
 627
 628         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
 629                            &dev_eqn, sizeof(dev_eqn)))
 630                 return -EFAULT;
 631
 632         return 0;
 633 }
 634
 635 /*
 636  *Security note:
 637  * The hardware protection mechanism works like this: Each device object that
 638  * is subject to UAR doorbells (QP/SQ/CQ) gets a UAR ID (called uar_page in
 639  * the device specification manual) upon its creation. Then upon doorbell,
 640  * hardware fetches the object context for which the doorbell was rang, and
 641  * validates that the UAR through which the DB was rang matches the UAR ID
 642  * of the object.
 643  * If no match the doorbell is silently ignored by the hardware. Of course,
 644  * the user cannot ring a doorbell on a UAR that was not mapped to it.
 645  * Now in devx, as the devx kernel does not manipulate the QP/SQ/CQ command
 646  * mailboxes (except tagging them with UID), we expose to the user its UAR
 647  * ID, so it can embed it in these objects in the expected specification
 648  * format. So the only thing the user can do is hurt itself by creating a
 649  * QP/SQ/CQ with a UAR ID other than his, and then in this case other users
 650  * may ring a doorbell on its objects.
 651  * The consequence of that will be that another user can schedule a QP/SQ
 652  * of the buggy user for execution (just insert it to the hardware schedule
 653  * queue or arm its CQ for event generation), no further harm is expected.
 654  */
 655 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_QUERY_UAR)(
 656         struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
 657 {
 658         struct mlx5_ib_ucontext *c;
 659         struct mlx5_ib_dev *dev;
 660         u32 user_idx;
 661         s32 dev_idx;
 662
 663         c = devx_ufile2uctx(file);
 664         if (IS_ERR(c))
 665                 return PTR_ERR(c);
 666         dev = to_mdev(c->ibucontext.device);
 667
 668         if (uverbs_copy_from(&user_idx, attrs,
 669                              MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX))
 670                 return -EFAULT;
 671
 672         dev_idx = bfregn_to_uar_index(dev, &c->bfregi, user_idx, true);
 673         if (dev_idx < 0)
 674                 return dev_idx;
 675
 676         if (uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
 677                            &dev_idx, sizeof(dev_idx)))
 678                 return -EFAULT;
 679
 680         return 0;
 681 }
 682
 683 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OTHER)(
 684         struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
 685 {
 686         struct mlx5_ib_ucontext *c;
 687         struct mlx5_ib_dev *dev;
 688         void *cmd_in = uverbs_attr_get_alloced_ptr(
 689                 attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN);
 690         int cmd_out_len = uverbs_attr_get_len(attrs,
 691                                         MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT);
 692         void *cmd_out;
 693         int err;
 694         int uid;
 695
 696         c = devx_ufile2uctx(file);
 697         if (IS_ERR(c))
 698                 return PTR_ERR(c);
 699         dev = to_mdev(c->ibucontext.device);
 700
 701         uid = devx_get_uid(c, cmd_in);
 702         if (uid < 0)
 703                 return uid;
 704
 705         /* Only white list of some general HCA commands are allowed for this method. */
 706         if (!devx_is_general_cmd(cmd_in))
 707                 return -EINVAL;
 708
 709         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
 710         if (IS_ERR(cmd_out))
 711                 return PTR_ERR(cmd_out);
 712
 713         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
 714         err = mlx5_cmd_exec(dev->mdev, cmd_in,
 715                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_IN),
 716                             cmd_out, cmd_out_len);
 717         if (err)
 718                 return err;
 719
 720         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT, cmd_out,
 721                               cmd_out_len);
 722 }
 723
 724 static void devx_obj_build_destroy_cmd(void *in, void *out, void *din,
 725                                        u32 *dinlen,
 726                                        u32 *obj_id)
 727 {
 728         u16 obj_type = MLX5_GET(general_obj_in_cmd_hdr, in, obj_type);
 729         u16 uid = MLX5_GET(general_obj_in_cmd_hdr, in, uid);
 730
 731         *obj_id = MLX5_GET(general_obj_out_cmd_hdr, out, obj_id);
 732         *dinlen = MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr);
 733
 734         MLX5_SET(general_obj_in_cmd_hdr, din, obj_id, *obj_id);
 735         MLX5_SET(general_obj_in_cmd_hdr, din, uid, uid);
 736
 737         switch (MLX5_GET(general_obj_in_cmd_hdr, in, opcode)) {
 738         case MLX5_CMD_OP_CREATE_GENERAL_OBJECT:
 739                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_GENERAL_OBJECT);
 740                 MLX5_SET(general_obj_in_cmd_hdr, din, obj_type, obj_type);
 741                 break;
 742
 743         case MLX5_CMD_OP_CREATE_MKEY:
 744                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_MKEY);
 745                 break;
 746         case MLX5_CMD_OP_CREATE_CQ:
 747                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_CQ);
 748                 break;
 749         case MLX5_CMD_OP_ALLOC_PD:
 750                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_PD);
 751                 break;
 752         case MLX5_CMD_OP_ALLOC_TRANSPORT_DOMAIN:
 753                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 754                          MLX5_CMD_OP_DEALLOC_TRANSPORT_DOMAIN);
 755                 break;
 756         case MLX5_CMD_OP_CREATE_RMP:
 757                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RMP);
 758                 break;
 759         case MLX5_CMD_OP_CREATE_SQ:
 760                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SQ);
 761                 break;
 762         case MLX5_CMD_OP_CREATE_RQ:
 763                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQ);
 764                 break;
 765         case MLX5_CMD_OP_CREATE_RQT:
 766                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_RQT);
 767                 break;
 768         case MLX5_CMD_OP_CREATE_TIR:
 769                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIR);
 770                 break;
 771         case MLX5_CMD_OP_CREATE_TIS:
 772                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_TIS);
 773                 break;
 774         case MLX5_CMD_OP_ALLOC_Q_COUNTER:
 775                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 776                          MLX5_CMD_OP_DEALLOC_Q_COUNTER);
 777                 break;
 778         case MLX5_CMD_OP_CREATE_FLOW_TABLE:
 779                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_table_in);
 780                 *obj_id = MLX5_GET(create_flow_table_out, out, table_id);
 781                 MLX5_SET(destroy_flow_table_in, din, other_vport,
 782                          MLX5_GET(create_flow_table_in,  in, other_vport));
 783                 MLX5_SET(destroy_flow_table_in, din, vport_number,
 784                          MLX5_GET(create_flow_table_in,  in, vport_number));
 785                 MLX5_SET(destroy_flow_table_in, din, table_type,
 786                          MLX5_GET(create_flow_table_in,  in, table_type));
 787                 MLX5_SET(destroy_flow_table_in, din, table_id, *obj_id);
 788                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 789                          MLX5_CMD_OP_DESTROY_FLOW_TABLE);
 790                 break;
 791         case MLX5_CMD_OP_CREATE_FLOW_GROUP:
 792                 *dinlen = MLX5_ST_SZ_BYTES(destroy_flow_group_in);
 793                 *obj_id = MLX5_GET(create_flow_group_out, out, group_id);
 794                 MLX5_SET(destroy_flow_group_in, din, other_vport,
 795                          MLX5_GET(create_flow_group_in, in, other_vport));
 796                 MLX5_SET(destroy_flow_group_in, din, vport_number,
 797                          MLX5_GET(create_flow_group_in, in, vport_number));
 798                 MLX5_SET(destroy_flow_group_in, din, table_type,
 799                          MLX5_GET(create_flow_group_in, in, table_type));
 800                 MLX5_SET(destroy_flow_group_in, din, table_id,
 801                          MLX5_GET(create_flow_group_in, in, table_id));
 802                 MLX5_SET(destroy_flow_group_in, din, group_id, *obj_id);
 803                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 804                          MLX5_CMD_OP_DESTROY_FLOW_GROUP);
 805                 break;
 806         case MLX5_CMD_OP_SET_FLOW_TABLE_ENTRY:
 807                 *dinlen = MLX5_ST_SZ_BYTES(delete_fte_in);
 808                 *obj_id = MLX5_GET(set_fte_in, in, flow_index);
 809                 MLX5_SET(delete_fte_in, din, other_vport,
 810                          MLX5_GET(set_fte_in,  in, other_vport));
 811                 MLX5_SET(delete_fte_in, din, vport_number,
 812                          MLX5_GET(set_fte_in, in, vport_number));
 813                 MLX5_SET(delete_fte_in, din, table_type,
 814                          MLX5_GET(set_fte_in, in, table_type));
 815                 MLX5_SET(delete_fte_in, din, table_id,
 816                          MLX5_GET(set_fte_in, in, table_id));
 817                 MLX5_SET(delete_fte_in, din, flow_index, *obj_id);
 818                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 819                          MLX5_CMD_OP_DELETE_FLOW_TABLE_ENTRY);
 820                 break;
 821         case MLX5_CMD_OP_ALLOC_FLOW_COUNTER:
 822                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 823                          MLX5_CMD_OP_DEALLOC_FLOW_COUNTER);
 824                 break;
 825         case MLX5_CMD_OP_ALLOC_PACKET_REFORMAT_CONTEXT:
 826                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 827                          MLX5_CMD_OP_DEALLOC_PACKET_REFORMAT_CONTEXT);
 828                 break;
 829         case MLX5_CMD_OP_ALLOC_MODIFY_HEADER_CONTEXT:
 830                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 831                          MLX5_CMD_OP_DEALLOC_MODIFY_HEADER_CONTEXT);
 832                 break;
 833         case MLX5_CMD_OP_CREATE_SCHEDULING_ELEMENT:
 834                 *dinlen = MLX5_ST_SZ_BYTES(destroy_scheduling_element_in);
 835                 *obj_id = MLX5_GET(create_scheduling_element_out, out,
 836                                    scheduling_element_id);
 837                 MLX5_SET(destroy_scheduling_element_in, din,
 838                          scheduling_hierarchy,
 839                          MLX5_GET(create_scheduling_element_in, in,
 840                                   scheduling_hierarchy));
 841                 MLX5_SET(destroy_scheduling_element_in, din,
 842                          scheduling_element_id, *obj_id);
 843                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 844                          MLX5_CMD_OP_DESTROY_SCHEDULING_ELEMENT);
 845                 break;
 846         case MLX5_CMD_OP_ADD_VXLAN_UDP_DPORT:
 847                 *dinlen = MLX5_ST_SZ_BYTES(delete_vxlan_udp_dport_in);
 848                 *obj_id = MLX5_GET(add_vxlan_udp_dport_in, in, vxlan_udp_port);
 849                 MLX5_SET(delete_vxlan_udp_dport_in, din, vxlan_udp_port, *obj_id);
 850                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 851                          MLX5_CMD_OP_DELETE_VXLAN_UDP_DPORT);
 852                 break;
 853         case MLX5_CMD_OP_SET_L2_TABLE_ENTRY:
 854                 *dinlen = MLX5_ST_SZ_BYTES(delete_l2_table_entry_in);
 855                 *obj_id = MLX5_GET(set_l2_table_entry_in, in, table_index);
 856                 MLX5_SET(delete_l2_table_entry_in, din, table_index, *obj_id);
 857                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 858                          MLX5_CMD_OP_DELETE_L2_TABLE_ENTRY);
 859                 break;
 860         case MLX5_CMD_OP_CREATE_QP:
 861                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_QP);
 862                 break;
 863         case MLX5_CMD_OP_CREATE_SRQ:
 864                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_SRQ);
 865                 break;
 866         case MLX5_CMD_OP_CREATE_XRC_SRQ:
 867                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode,
 868                          MLX5_CMD_OP_DESTROY_XRC_SRQ);
 869                 break;
 870         case MLX5_CMD_OP_CREATE_DCT:
 871                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_DCT);
 872                 break;
 873         case MLX5_CMD_OP_CREATE_XRQ:
 874                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DESTROY_XRQ);
 875                 break;
 876         case MLX5_CMD_OP_ATTACH_TO_MCG:
 877                 *dinlen = MLX5_ST_SZ_BYTES(detach_from_mcg_in);
 878                 MLX5_SET(detach_from_mcg_in, din, qpn,
 879                          MLX5_GET(attach_to_mcg_in, in, qpn));
 880                 memcpy(MLX5_ADDR_OF(detach_from_mcg_in, din, multicast_gid),
 881                        MLX5_ADDR_OF(attach_to_mcg_in, in, multicast_gid),
 882                        MLX5_FLD_SZ_BYTES(attach_to_mcg_in, multicast_gid));
 883                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DETACH_FROM_MCG);
 884                 break;
 885         case MLX5_CMD_OP_ALLOC_XRCD:
 886                 MLX5_SET(general_obj_in_cmd_hdr, din, opcode, MLX5_CMD_OP_DEALLOC_XRCD);
 887                 break;
 888         default:
 889                 /* The entry must match to one of the devx_is_obj_create_cmd */
 890                 WARN_ON(true);
 891                 break;
 892         }
 893 }
 894
 895 static int devx_obj_cleanup(struct ib_uobject *uobject,
 896                             enum rdma_remove_reason why)
 897 {
 898         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
 899         struct devx_obj *obj = uobject->object;
 900         int ret;
 901
 902         ret = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
 903         if (ib_is_destroy_retryable(ret, why, uobject))
 904                 return ret;
 905
 906         kfree(obj);
 907         return ret;
 908 }
 909
 910 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_CREATE)(
 911         struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
 912 {
 913         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN);
 914         int cmd_out_len =  uverbs_attr_get_len(attrs,
 915                                         MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT);
 916         void *cmd_out;
 917         struct ib_uobject *uobj = uverbs_attr_get_uobject(
 918                 attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE);
 919         struct mlx5_ib_ucontext *c = to_mucontext(uobj->context);
 920         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
 921         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
 922         struct devx_obj *obj;
 923         int err;
 924         int uid;
 925         u32 obj_id;
 926         u16 opcode;
 927
 928         uid = devx_get_uid(c, cmd_in);
 929         if (uid < 0)
 930                 return uid;
 931
 932         if (!devx_is_obj_create_cmd(cmd_in, &opcode))
 933                 return -EINVAL;
 934
 935         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
 936         if (IS_ERR(cmd_out))
 937                 return PTR_ERR(cmd_out);
 938
 939         obj = kzalloc(sizeof(struct devx_obj), GFP_KERNEL);
 940         if (!obj)
 941                 return -ENOMEM;
 942
 943         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
 944         devx_set_umem_valid(cmd_in);
 945
 946         err = mlx5_cmd_exec(dev->mdev, cmd_in,
 947                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN),
 948                             cmd_out, cmd_out_len);
 949         if (err)
 950                 goto obj_free;
 951
 952         uobj->object = obj;
 953         obj->mdev = dev->mdev;
 954         devx_obj_build_destroy_cmd(cmd_in, cmd_out, obj->dinbox, &obj->dinlen,
 955                                    &obj_id);
 956         WARN_ON(obj->dinlen > MLX5_MAX_DESTROY_INBOX_SIZE_DW * sizeof(u32));
 957
 958         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT, cmd_out, cmd_out_len);
 959         if (err)
 960                 goto obj_destroy;
 961
 962         obj->obj_id = get_enc_obj_id(opcode, obj_id);
 963         return 0;
 964
 965 obj_destroy:
 966         mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
 967 obj_free:
 968         kfree(obj);
 969         return err;
 970 }
 971
 972 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_MODIFY)(
 973         struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
 974 {
 975         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN);
 976         int cmd_out_len = uverbs_attr_get_len(attrs,
 977                                         MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT);
 978         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
 979                                                           MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE);
 980         struct mlx5_ib_ucontext *c = to_mucontext(uobj->context);
 981         struct devx_obj *obj = uobj->object;
 982         void *cmd_out;
 983         int err;
 984         int uid;
 985
 986         uid = devx_get_uid(c, cmd_in);
 987         if (uid < 0)
 988                 return uid;
 989
 990         if (!devx_is_obj_modify_cmd(cmd_in))
 991                 return -EINVAL;
 992
 993         if (!devx_is_valid_obj_id(obj, cmd_in))
 994                 return -EINVAL;
 995
 996         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
 997         if (IS_ERR(cmd_out))
 998                 return PTR_ERR(cmd_out);
 999
1000         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1001         devx_set_umem_valid(cmd_in);
1002
1003         err = mlx5_cmd_exec(obj->mdev, cmd_in,
1004                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN),
1005                             cmd_out, cmd_out_len);
1006         if (err)
1007                 return err;
1008
1009         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1010                               cmd_out, cmd_out_len);
1011 }
1012
1013 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_OBJ_QUERY)(
1014         struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
1015 {
1016         void *cmd_in = uverbs_attr_get_alloced_ptr(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN);
1017         int cmd_out_len = uverbs_attr_get_len(attrs,
1018                                               MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT);
1019         struct ib_uobject *uobj = uverbs_attr_get_uobject(attrs,
1020                                                           MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE);
1021         struct mlx5_ib_ucontext *c = to_mucontext(uobj->context);
1022         struct devx_obj *obj = uobj->object;
1023         void *cmd_out;
1024         int err;
1025         int uid;
1026
1027         uid = devx_get_uid(c, cmd_in);
1028         if (uid < 0)
1029                 return uid;
1030
1031         if (!devx_is_obj_query_cmd(cmd_in))
1032                 return -EINVAL;
1033
1034         if (!devx_is_valid_obj_id(obj, cmd_in))
1035                 return -EINVAL;
1036
1037         cmd_out = uverbs_zalloc(attrs, cmd_out_len);
1038         if (IS_ERR(cmd_out))
1039                 return PTR_ERR(cmd_out);
1040
1041         MLX5_SET(general_obj_in_cmd_hdr, cmd_in, uid, uid);
1042         err = mlx5_cmd_exec(obj->mdev, cmd_in,
1043                             uverbs_attr_get_len(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN),
1044                             cmd_out, cmd_out_len);
1045         if (err)
1046                 return err;
1047
1048         return uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1049                               cmd_out, cmd_out_len);
1050 }
1051
1052 static int devx_umem_get(struct mlx5_ib_dev *dev, struct ib_ucontext *ucontext,
1053                          struct uverbs_attr_bundle *attrs,
1054                          struct devx_umem *obj)
1055 {
1056         u64 addr;
1057         size_t size;
1058         u32 access;
1059         int npages;
1060         int err;
1061         u32 page_mask;
1062
1063         if (uverbs_copy_from(&addr, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR) ||
1064             uverbs_copy_from(&size, attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_LEN))
1065                 return -EFAULT;
1066
1067         err = uverbs_get_flags32(&access, attrs,
1068                                  MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
1069                                  IB_ACCESS_SUPPORTED);
1070         if (err)
1071                 return err;
1072
1073         err = ib_check_mr_access(access);
1074         if (err)
1075                 return err;
1076
1077         obj->umem = ib_umem_get(ucontext, addr, size, access, 0);
1078         if (IS_ERR(obj->umem))
1079                 return PTR_ERR(obj->umem);
1080
1081         mlx5_ib_cont_pages(obj->umem, obj->umem->address,
1082                            MLX5_MKEY_PAGE_SHIFT_MASK, &npages,
1083                            &obj->page_shift, &obj->ncont, NULL);
1084
1085         if (!npages) {
1086                 ib_umem_release(obj->umem);
1087                 return -EINVAL;
1088         }
1089
1090         page_mask = (1 << obj->page_shift) - 1;
1091         obj->page_offset = obj->umem->address & page_mask;
1092
1093         return 0;
1094 }
1095
1096 static int devx_umem_reg_cmd_alloc(struct uverbs_attr_bundle *attrs,
1097                                    struct devx_umem *obj,
1098                                    struct devx_umem_reg_cmd *cmd)
1099 {
1100         cmd->inlen = MLX5_ST_SZ_BYTES(create_umem_in) +
1101                     (MLX5_ST_SZ_BYTES(mtt) * obj->ncont);
1102         cmd->in = uverbs_zalloc(attrs, cmd->inlen);
1103         return PTR_ERR_OR_ZERO(cmd->in);
1104 }
1105
1106 static void devx_umem_reg_cmd_build(struct mlx5_ib_dev *dev,
1107                                     struct devx_umem *obj,
1108                                     struct devx_umem_reg_cmd *cmd)
1109 {
1110         void *umem;
1111         __be64 *mtt;
1112
1113         umem = MLX5_ADDR_OF(create_umem_in, cmd->in, umem);
1114         mtt = (__be64 *)MLX5_ADDR_OF(umem, umem, mtt);
1115
1116         MLX5_SET(general_obj_in_cmd_hdr, cmd->in, opcode, MLX5_CMD_OP_CREATE_GENERAL_OBJECT);
1117         MLX5_SET(general_obj_in_cmd_hdr, cmd->in, obj_type, MLX5_OBJ_TYPE_UMEM);
1118         MLX5_SET64(umem, umem, num_of_mtt, obj->ncont);
1119         MLX5_SET(umem, umem, log_page_size, obj->page_shift -
1120                                             MLX5_ADAPTER_PAGE_SHIFT);
1121         MLX5_SET(umem, umem, page_offset, obj->page_offset);
1122         mlx5_ib_populate_pas(dev, obj->umem, obj->page_shift, mtt,
1123                              (obj->umem->writable ? MLX5_IB_MTT_WRITE : 0) |
1124                              MLX5_IB_MTT_READ);
1125 }
1126
1127 static int UVERBS_HANDLER(MLX5_IB_METHOD_DEVX_UMEM_REG)(
1128         struct ib_uverbs_file *file, struct uverbs_attr_bundle *attrs)
1129 {
1130         struct devx_umem_reg_cmd cmd;
1131         struct devx_umem *obj;
1132         struct ib_uobject *uobj = uverbs_attr_get_uobject(
1133                 attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE);
1134         u32 obj_id;
1135         struct mlx5_ib_ucontext *c = to_mucontext(uobj->context);
1136         struct mlx5_ib_dev *dev = to_mdev(c->ibucontext.device);
1137         int err;
1138
1139         if (!c->devx_uid)
1140                 return -EINVAL;
1141
1142         if (!capable(CAP_NET_RAW))
1143                 return -EPERM;
1144
1145         obj = kzalloc(sizeof(struct devx_umem), GFP_KERNEL);
1146         if (!obj)
1147                 return -ENOMEM;
1148
1149         err = devx_umem_get(dev, &c->ibucontext, attrs, obj);
1150         if (err)
1151                 goto err_obj_free;
1152
1153         err = devx_umem_reg_cmd_alloc(attrs, obj, &cmd);
1154         if (err)
1155                 goto err_umem_release;
1156
1157         devx_umem_reg_cmd_build(dev, obj, &cmd);
1158
1159         MLX5_SET(general_obj_in_cmd_hdr, cmd.in, uid, c->devx_uid);
1160         err = mlx5_cmd_exec(dev->mdev, cmd.in, cmd.inlen, cmd.out,
1161                             sizeof(cmd.out));
1162         if (err)
1163                 goto err_umem_release;
1164
1165         obj->mdev = dev->mdev;
1166         uobj->object = obj;
1167         devx_obj_build_destroy_cmd(cmd.in, cmd.out, obj->dinbox, &obj->dinlen, &obj_id);
1168         err = uverbs_copy_to(attrs, MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID, &obj_id, sizeof(obj_id));
1169         if (err)
1170                 goto err_umem_destroy;
1171
1172         return 0;
1173
1174 err_umem_destroy:
1175         mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, cmd.out, sizeof(cmd.out));
1176 err_umem_release:
1177         ib_umem_release(obj->umem);
1178 err_obj_free:
1179         kfree(obj);
1180         return err;
1181 }
1182
1183 static int devx_umem_cleanup(struct ib_uobject *uobject,
1184                              enum rdma_remove_reason why)
1185 {
1186         struct devx_umem *obj = uobject->object;
1187         u32 out[MLX5_ST_SZ_DW(general_obj_out_cmd_hdr)];
1188         int err;
1189
1190         err = mlx5_cmd_exec(obj->mdev, obj->dinbox, obj->dinlen, out, sizeof(out));
1191         if (ib_is_destroy_retryable(err, why, uobject))
1192                 return err;
1193
1194         ib_umem_release(obj->umem);
1195         kfree(obj);
1196         return 0;
1197 }
1198
1199 DECLARE_UVERBS_NAMED_METHOD(
1200         MLX5_IB_METHOD_DEVX_UMEM_REG,
1201         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_REG_HANDLE,
1202                         MLX5_IB_OBJECT_DEVX_UMEM,
1203                         UVERBS_ACCESS_NEW,
1204                         UA_MANDATORY),
1205         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ADDR,
1206                            UVERBS_ATTR_TYPE(u64),
1207                            UA_MANDATORY),
1208         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_LEN,
1209                            UVERBS_ATTR_TYPE(u64),
1210                            UA_MANDATORY),
1211         UVERBS_ATTR_FLAGS_IN(MLX5_IB_ATTR_DEVX_UMEM_REG_ACCESS,
1212                              enum ib_access_flags),
1213         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_UMEM_REG_OUT_ID,
1214                             UVERBS_ATTR_TYPE(u32),
1215                             UA_MANDATORY));
1216
1217 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
1218         MLX5_IB_METHOD_DEVX_UMEM_DEREG,
1219         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_UMEM_DEREG_HANDLE,
1220                         MLX5_IB_OBJECT_DEVX_UMEM,
1221                         UVERBS_ACCESS_DESTROY,
1222                         UA_MANDATORY));
1223
1224 DECLARE_UVERBS_NAMED_METHOD(
1225         MLX5_IB_METHOD_DEVX_QUERY_EQN,
1226         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_EQN_USER_VEC,
1227                            UVERBS_ATTR_TYPE(u32),
1228                            UA_MANDATORY),
1229         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_EQN_DEV_EQN,
1230                             UVERBS_ATTR_TYPE(u32),
1231                             UA_MANDATORY));
1232
1233 DECLARE_UVERBS_NAMED_METHOD(
1234         MLX5_IB_METHOD_DEVX_QUERY_UAR,
1235         UVERBS_ATTR_PTR_IN(MLX5_IB_ATTR_DEVX_QUERY_UAR_USER_IDX,
1236                            UVERBS_ATTR_TYPE(u32),
1237                            UA_MANDATORY),
1238         UVERBS_ATTR_PTR_OUT(MLX5_IB_ATTR_DEVX_QUERY_UAR_DEV_IDX,
1239                             UVERBS_ATTR_TYPE(u32),
1240                             UA_MANDATORY));
1241
1242 DECLARE_UVERBS_NAMED_METHOD(
1243         MLX5_IB_METHOD_DEVX_OTHER,
1244         UVERBS_ATTR_PTR_IN(
1245                 MLX5_IB_ATTR_DEVX_OTHER_CMD_IN,
1246                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1247                 UA_MANDATORY,
1248                 UA_ALLOC_AND_COPY),
1249         UVERBS_ATTR_PTR_OUT(
1250                 MLX5_IB_ATTR_DEVX_OTHER_CMD_OUT,
1251                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1252                 UA_MANDATORY));
1253
1254 DECLARE_UVERBS_NAMED_METHOD(
1255         MLX5_IB_METHOD_DEVX_OBJ_CREATE,
1256         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_CREATE_HANDLE,
1257                         MLX5_IB_OBJECT_DEVX_OBJ,
1258                         UVERBS_ACCESS_NEW,
1259                         UA_MANDATORY),
1260         UVERBS_ATTR_PTR_IN(
1261                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_IN,
1262                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1263                 UA_MANDATORY,
1264                 UA_ALLOC_AND_COPY),
1265         UVERBS_ATTR_PTR_OUT(
1266                 MLX5_IB_ATTR_DEVX_OBJ_CREATE_CMD_OUT,
1267                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1268                 UA_MANDATORY));
1269
1270 DECLARE_UVERBS_NAMED_METHOD_DESTROY(
1271         MLX5_IB_METHOD_DEVX_OBJ_DESTROY,
1272         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_DESTROY_HANDLE,
1273                         MLX5_IB_OBJECT_DEVX_OBJ,
1274                         UVERBS_ACCESS_DESTROY,
1275                         UA_MANDATORY));
1276
1277 DECLARE_UVERBS_NAMED_METHOD(
1278         MLX5_IB_METHOD_DEVX_OBJ_MODIFY,
1279         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_MODIFY_HANDLE,
1280                         MLX5_IB_OBJECT_DEVX_OBJ,
1281                         UVERBS_ACCESS_WRITE,
1282                         UA_MANDATORY),
1283         UVERBS_ATTR_PTR_IN(
1284                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_IN,
1285                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1286                 UA_MANDATORY,
1287                 UA_ALLOC_AND_COPY),
1288         UVERBS_ATTR_PTR_OUT(
1289                 MLX5_IB_ATTR_DEVX_OBJ_MODIFY_CMD_OUT,
1290                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1291                 UA_MANDATORY));
1292
1293 DECLARE_UVERBS_NAMED_METHOD(
1294         MLX5_IB_METHOD_DEVX_OBJ_QUERY,
1295         UVERBS_ATTR_IDR(MLX5_IB_ATTR_DEVX_OBJ_QUERY_HANDLE,
1296                         MLX5_IB_OBJECT_DEVX_OBJ,
1297                         UVERBS_ACCESS_READ,
1298                         UA_MANDATORY),
1299         UVERBS_ATTR_PTR_IN(
1300                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_IN,
1301                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_in_cmd_hdr)),
1302                 UA_MANDATORY,
1303                 UA_ALLOC_AND_COPY),
1304         UVERBS_ATTR_PTR_OUT(
1305                 MLX5_IB_ATTR_DEVX_OBJ_QUERY_CMD_OUT,
1306                 UVERBS_ATTR_MIN_SIZE(MLX5_ST_SZ_BYTES(general_obj_out_cmd_hdr)),
1307                 UA_MANDATORY));
1308
1309 DECLARE_UVERBS_GLOBAL_METHODS(MLX5_IB_OBJECT_DEVX,
1310                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OTHER),
1311                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_UAR),
1312                               &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_QUERY_EQN));
1313
1314 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_OBJ,
1315                             UVERBS_TYPE_ALLOC_IDR(devx_obj_cleanup),
1316                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_CREATE),
1317                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_DESTROY),
1318                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_MODIFY),
1319                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_OBJ_QUERY));
1320
1321 DECLARE_UVERBS_NAMED_OBJECT(MLX5_IB_OBJECT_DEVX_UMEM,
1322                             UVERBS_TYPE_ALLOC_IDR(devx_umem_cleanup),
1323                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_REG),
1324                             &UVERBS_METHOD(MLX5_IB_METHOD_DEVX_UMEM_DEREG));
1325
1326 static bool devx_is_supported(struct ib_device *device)
1327 {
1328         struct mlx5_ib_dev *dev = to_mdev(device);
1329
1330         return !dev->rep && MLX5_CAP_GEN_64(dev->mdev, general_obj_types) &
1331                                     MLX5_GENERAL_OBJ_TYPES_CAP_UCTX;
1332 }
1333
1334 const struct uapi_definition mlx5_ib_devx_defs[] = {
1335         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1336                 MLX5_IB_OBJECT_DEVX,
1337                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1338         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1339                 MLX5_IB_OBJECT_DEVX_OBJ,
1340                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1341         UAPI_DEF_CHAIN_OBJ_TREE_NAMED(
1342                 MLX5_IB_OBJECT_DEVX_UMEM,
1343                 UAPI_DEF_IS_OBJ_SUPPORTED(devx_is_supported)),
1344         {},
1345 };