drivers/infiniband/core/uverbs_uapi.c

   1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
   2 /*
   3  * Copyright (c) 2017, Mellanox Technologies inc.  All rights reserved.
   4  */
   5 #include <rdma/uverbs_ioctl.h>
   6 #include <rdma/rdma_user_ioctl.h>
   7 #include <linux/bitops.h>
   8 #include "rdma_core.h"
   9 #include "uverbs.h"
  10
  11 static ssize_t ib_uverbs_notsupp(struct uverbs_attr_bundle *attrs,
  12                                  const char __user *buf, int in_len,
  13                                  int out_len)
  14 {
  15         return -EOPNOTSUPP;
  16 }
  17
  18 static int ib_uverbs_ex_notsupp(struct uverbs_attr_bundle *attrs,
  19                                 struct ib_udata *ucore, struct ib_udata *uhw)
  20 {
  21         return -EOPNOTSUPP;
  22 }
  23
  24 static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
  25 {
  26         void *elm;
  27         int rc;
  28
  29         if (key == UVERBS_API_KEY_ERR)
  30                 return ERR_PTR(-EOVERFLOW);
  31
  32         elm = kzalloc(alloc_size, GFP_KERNEL);
  33         rc = radix_tree_insert(&uapi->radix, key, elm);
  34         if (rc) {
  35                 kfree(elm);
  36                 return ERR_PTR(rc);
  37         }
  38
  39         return elm;
  40 }
  41
  42 static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,
  43                               size_t alloc_size, bool *exists)
  44 {
  45         void *elm;
  46
  47         elm = uapi_add_elm(uapi, key, alloc_size);
  48         if (!IS_ERR(elm)) {
  49                 *exists = false;
  50                 return elm;
  51         }
  52
  53         if (elm != ERR_PTR(-EEXIST))
  54                 return elm;
  55
  56         elm = radix_tree_lookup(&uapi->radix, key);
  57         if (WARN_ON(!elm))
  58                 return ERR_PTR(-EINVAL);
  59         *exists = true;
  60         return elm;
  61 }
  62
  63 static int uapi_create_write(struct uverbs_api *uapi,
  64                              struct ib_device *ibdev,
  65                              const struct uapi_definition *def,
  66                              u32 obj_key,
  67                              u32 *cur_method_key)
  68 {
  69         struct uverbs_api_write_method *method_elm;
  70         u32 method_key = obj_key;
  71         bool exists;
  72
  73         if (def->write.is_ex)
  74                 method_key |= uapi_key_write_ex_method(def->write.command_num);
  75         else
  76                 method_key |= uapi_key_write_method(def->write.command_num);
  77
  78         method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
  79                                       &exists);
  80         if (IS_ERR(method_elm))
  81                 return PTR_ERR(method_elm);
  82
  83         if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex ||
  84                                method_elm->handler_ex || method_elm->handler)))
  85                 return -EINVAL;
  86
  87         method_elm->is_ex = def->write.is_ex;
  88         if (def->write.is_ex) {
  89                 method_elm->handler_ex = def->func_write_ex;
  90
  91                 method_elm->disabled = !(ibdev->uverbs_ex_cmd_mask &
  92                                          BIT_ULL(def->write.command_num));
  93         } else {
  94                 method_elm->handler = def->func_write;
  95
  96                 method_elm->disabled = !(ibdev->uverbs_cmd_mask &
  97                                          BIT_ULL(def->write.command_num));
  98         }
  99
 100         *cur_method_key = method_key;
 101         return 0;
 102 }
 103
 104 static int uapi_merge_method(struct uverbs_api *uapi,
 105                              struct uverbs_api_object *obj_elm, u32 obj_key,
 106                              const struct uverbs_method_def *method,
 107                              bool is_driver)
 108 {
 109         u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
 110         struct uverbs_api_ioctl_method *method_elm;
 111         unsigned int i;
 112         bool exists;
 113
 114         if (!method->attrs)
 115                 return 0;
 116
 117         method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
 118                                       &exists);
 119         if (IS_ERR(method_elm))
 120                 return PTR_ERR(method_elm);
 121         if (exists) {
 122                 /*
 123                  * This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE
 124                  */
 125                 if (WARN_ON(method->handler))
 126                         return -EINVAL;
 127         } else {
 128                 WARN_ON(!method->handler);
 129                 rcu_assign_pointer(method_elm->handler, method->handler);
 130                 if (method->handler != uverbs_destroy_def_handler)
 131                         method_elm->driver_method = is_driver;
 132         }
 133
 134         for (i = 0; i != method->num_attrs; i++) {
 135                 const struct uverbs_attr_def *attr = (*method->attrs)[i];
 136                 struct uverbs_api_attr *attr_slot;
 137
 138                 if (!attr)
 139                         continue;
 140
 141                 /*
 142                  * ENUM_IN contains the 'ids' pointer to the driver's .rodata,
 143                  * so if it is specified by a driver then it always makes this
 144                  * into a driver method.
 145                  */
 146                 if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
 147                         method_elm->driver_method |= is_driver;
 148
 149                 /*
 150                  * Like other uobject based things we only support a single
 151                  * uobject being NEW'd or DESTROY'd
 152                  */
 153                 if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
 154                         u8 access = attr->attr.u2.objs_arr.access;
 155
 156                         if (WARN_ON(access == UVERBS_ACCESS_NEW ||
 157                                     access == UVERBS_ACCESS_DESTROY))
 158                                 return -EINVAL;
 159                 }
 160
 161                 attr_slot =
 162                         uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
 163                                      sizeof(*attr_slot));
 164                 /* Attributes are not allowed to be modified by drivers */
 165                 if (IS_ERR(attr_slot))
 166                         return PTR_ERR(attr_slot);
 167
 168                 attr_slot->spec = attr->attr;
 169         }
 170
 171         return 0;
 172 }
 173
 174 static int uapi_merge_obj_tree(struct uverbs_api *uapi,
 175                                const struct uverbs_object_def *obj,
 176                                bool is_driver)
 177 {
 178         struct uverbs_api_object *obj_elm;
 179         unsigned int i;
 180         u32 obj_key;
 181         bool exists;
 182         int rc;
 183
 184         obj_key = uapi_key_obj(obj->id);
 185         obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);
 186         if (IS_ERR(obj_elm))
 187                 return PTR_ERR(obj_elm);
 188
 189         if (obj->type_attrs) {
 190                 if (WARN_ON(obj_elm->type_attrs))
 191                         return -EINVAL;
 192
 193                 obj_elm->type_attrs = obj->type_attrs;
 194                 obj_elm->type_class = obj->type_attrs->type_class;
 195                 /*
 196                  * Today drivers are only permitted to use idr_class
 197                  * types. They cannot use FD types because we currently have
 198                  * no way to revoke the fops pointer after device
 199                  * disassociation.
 200                  */
 201                 if (WARN_ON(is_driver &&
 202                             obj->type_attrs->type_class != &uverbs_idr_class))
 203                         return -EINVAL;
 204         }
 205
 206         if (!obj->methods)
 207                 return 0;
 208
 209         for (i = 0; i != obj->num_methods; i++) {
 210                 const struct uverbs_method_def *method = (*obj->methods)[i];
 211
 212                 if (!method)
 213                         continue;
 214
 215                 rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
 216                                        is_driver);
 217                 if (rc)
 218                         return rc;
 219         }
 220
 221         return 0;
 222 }
 223
 224 static int uapi_disable_elm(struct uverbs_api *uapi,
 225                             const struct uapi_definition *def,
 226                             u32 obj_key,
 227                             u32 method_key)
 228 {
 229         bool exists;
 230
 231         if (def->scope == UAPI_SCOPE_OBJECT) {
 232                 struct uverbs_api_object *obj_elm;
 233
 234                 obj_elm = uapi_add_get_elm(
 235                         uapi, obj_key, sizeof(*obj_elm), &exists);
 236                 if (IS_ERR(obj_elm))
 237                         return PTR_ERR(obj_elm);
 238                 obj_elm->disabled = 1;
 239                 return 0;
 240         }
 241
 242         if (def->scope == UAPI_SCOPE_METHOD &&
 243             uapi_key_is_ioctl_method(method_key)) {
 244                 struct uverbs_api_ioctl_method *method_elm;
 245
 246                 method_elm = uapi_add_get_elm(uapi, method_key,
 247                                               sizeof(*method_elm), &exists);
 248                 if (IS_ERR(method_elm))
 249                         return PTR_ERR(method_elm);
 250                 method_elm->disabled = 1;
 251                 return 0;
 252         }
 253
 254         if (def->scope == UAPI_SCOPE_METHOD &&
 255             (uapi_key_is_write_method(method_key) ||
 256              uapi_key_is_write_ex_method(method_key))) {
 257                 struct uverbs_api_write_method *write_elm;
 258
 259                 write_elm = uapi_add_get_elm(uapi, method_key,
 260                                              sizeof(*write_elm), &exists);
 261                 if (IS_ERR(write_elm))
 262                         return PTR_ERR(write_elm);
 263                 write_elm->disabled = 1;
 264                 return 0;
 265         }
 266
 267         WARN_ON(true);
 268         return -EINVAL;
 269 }
 270
 271 static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,
 272                           const struct uapi_definition *def_list,
 273                           bool is_driver)
 274 {
 275         const struct uapi_definition *def = def_list;
 276         u32 cur_obj_key = UVERBS_API_KEY_ERR;
 277         u32 cur_method_key = UVERBS_API_KEY_ERR;
 278         bool exists;
 279         int rc;
 280
 281         if (!def_list)
 282                 return 0;
 283
 284         for (;; def++) {
 285                 switch ((enum uapi_definition_kind)def->kind) {
 286                 case UAPI_DEF_CHAIN:
 287                         rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);
 288                         if (rc)
 289                                 return rc;
 290                         continue;
 291
 292                 case UAPI_DEF_CHAIN_OBJ_TREE:
 293                         if (WARN_ON(def->object_start.object_id !=
 294                                     def->chain_obj_tree->id))
 295                                 return -EINVAL;
 296
 297                         cur_obj_key = uapi_key_obj(def->object_start.object_id);
 298                         rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,
 299                                                  is_driver);
 300                         if (rc)
 301                                 return rc;
 302                         continue;
 303
 304                 case UAPI_DEF_END:
 305                         return 0;
 306
 307                 case UAPI_DEF_IS_SUPPORTED_DEV_FN: {
 308                         void **ibdev_fn = (void *)ibdev + def->needs_fn_offset;
 309
 310                         if (*ibdev_fn)
 311                                 continue;
 312                         rc = uapi_disable_elm(
 313                                 uapi, def, cur_obj_key, cur_method_key);
 314                         if (rc)
 315                                 return rc;
 316                         continue;
 317                 }
 318
 319                 case UAPI_DEF_IS_SUPPORTED_FUNC:
 320                         if (def->func_is_supported(ibdev))
 321                                 continue;
 322                         rc = uapi_disable_elm(
 323                                 uapi, def, cur_obj_key, cur_method_key);
 324                         if (rc)
 325                                 return rc;
 326                         continue;
 327
 328                 case UAPI_DEF_OBJECT_START: {
 329                         struct uverbs_api_object *obj_elm;
 330
 331                         cur_obj_key = uapi_key_obj(def->object_start.object_id);
 332                         obj_elm = uapi_add_get_elm(uapi, cur_obj_key,
 333                                                    sizeof(*obj_elm), &exists);
 334                         if (IS_ERR(obj_elm))
 335                                 return PTR_ERR(obj_elm);
 336                         continue;
 337                 }
 338
 339                 case UAPI_DEF_WRITE:
 340                         rc = uapi_create_write(
 341                                 uapi, ibdev, def, cur_obj_key, &cur_method_key);
 342                         if (rc)
 343                                 return rc;
 344                         continue;
 345                 }
 346                 WARN_ON(true);
 347                 return -EINVAL;
 348         }
 349 }
 350
 351 static int
 352 uapi_finalize_ioctl_method(struct uverbs_api *uapi,
 353                            struct uverbs_api_ioctl_method *method_elm,
 354                            u32 method_key)
 355 {
 356         struct radix_tree_iter iter;
 357         unsigned int num_attrs = 0;
 358         unsigned int max_bkey = 0;
 359         bool single_uobj = false;
 360         void __rcu **slot;
 361
 362         method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
 363         radix_tree_for_each_slot (slot, &uapi->radix, &iter,
 364                                   uapi_key_attrs_start(method_key)) {
 365                 struct uverbs_api_attr *elm =
 366                         rcu_dereference_protected(*slot, true);
 367                 u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
 368                 u32 attr_bkey = uapi_bkey_attr(attr_key);
 369                 u8 type = elm->spec.type;
 370
 371                 if (uapi_key_attr_to_ioctl_method(iter.index) !=
 372                     uapi_key_attr_to_ioctl_method(method_key))
 373                         break;
 374
 375                 if (elm->spec.mandatory)
 376                         __set_bit(attr_bkey, method_elm->attr_mandatory);
 377
 378                 if (type == UVERBS_ATTR_TYPE_IDR ||
 379                     type == UVERBS_ATTR_TYPE_FD) {
 380                         u8 access = elm->spec.u.obj.access;
 381
 382                         /*
 383                          * Verbs specs may only have one NEW/DESTROY, we don't
 384                          * have the infrastructure to abort multiple NEW's or
 385                          * cope with multiple DESTROY failure.
 386                          */
 387                         if (access == UVERBS_ACCESS_NEW ||
 388                             access == UVERBS_ACCESS_DESTROY) {
 389                                 if (WARN_ON(single_uobj))
 390                                         return -EINVAL;
 391
 392                                 single_uobj = true;
 393                                 if (WARN_ON(!elm->spec.mandatory))
 394                                         return -EINVAL;
 395                         }
 396
 397                         if (access == UVERBS_ACCESS_DESTROY)
 398                                 method_elm->destroy_bkey = attr_bkey;
 399                 }
 400
 401                 max_bkey = max(max_bkey, attr_bkey);
 402                 num_attrs++;
 403         }
 404
 405         method_elm->key_bitmap_len = max_bkey + 1;
 406         WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);
 407
 408         uapi_compute_bundle_size(method_elm, num_attrs);
 409         return 0;
 410 }
 411
 412 static int uapi_finalize(struct uverbs_api *uapi)
 413 {
 414         const struct uverbs_api_write_method **data;
 415         unsigned long max_write_ex = 0;
 416         unsigned long max_write = 0;
 417         struct radix_tree_iter iter;
 418         void __rcu **slot;
 419         int rc;
 420         int i;
 421
 422         radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
 423                 struct uverbs_api_ioctl_method *method_elm =
 424                         rcu_dereference_protected(*slot, true);
 425
 426                 if (uapi_key_is_ioctl_method(iter.index)) {
 427                         rc = uapi_finalize_ioctl_method(uapi, method_elm,
 428                                                         iter.index);
 429                         if (rc)
 430                                 return rc;
 431                 }
 432
 433                 if (uapi_key_is_write_method(iter.index))
 434                         max_write = max(max_write,
 435                                         iter.index & UVERBS_API_ATTR_KEY_MASK);
 436                 if (uapi_key_is_write_ex_method(iter.index))
 437                         max_write_ex =
 438                                 max(max_write_ex,
 439                                     iter.index & UVERBS_API_ATTR_KEY_MASK);
 440         }
 441
 442         uapi->notsupp_method.handler = ib_uverbs_notsupp;
 443         uapi->notsupp_method.handler_ex = ib_uverbs_ex_notsupp;
 444         uapi->num_write = max_write + 1;
 445         uapi->num_write_ex = max_write_ex + 1;
 446         data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
 447                              sizeof(*uapi->write_methods), GFP_KERNEL);
 448         for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
 449                 data[i] = &uapi->notsupp_method;
 450         uapi->write_methods = data;
 451         uapi->write_ex_methods = data + uapi->num_write;
 452
 453         radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
 454                 if (uapi_key_is_write_method(iter.index))
 455                         uapi->write_methods[iter.index &
 456                                             UVERBS_API_ATTR_KEY_MASK] =
 457                                 rcu_dereference_protected(*slot, true);
 458                 if (uapi_key_is_write_ex_method(iter.index))
 459                         uapi->write_ex_methods[iter.index &
 460                                                UVERBS_API_ATTR_KEY_MASK] =
 461                                 rcu_dereference_protected(*slot, true);
 462         }
 463
 464         return 0;
 465 }
 466
 467 static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)
 468 {
 469         struct radix_tree_iter iter;
 470         void __rcu **slot;
 471
 472         radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {
 473                 if (iter.index > last)
 474                         return;
 475                 kfree(rcu_dereference_protected(*slot, true));
 476                 radix_tree_iter_delete(&uapi->radix, &iter, slot);
 477         }
 478 }
 479
 480 static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)
 481 {
 482         uapi_remove_range(uapi, obj_key,
 483                           obj_key | UVERBS_API_METHOD_KEY_MASK |
 484                                   UVERBS_API_ATTR_KEY_MASK);
 485 }
 486
 487 static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)
 488 {
 489         uapi_remove_range(uapi, method_key,
 490                           method_key | UVERBS_API_ATTR_KEY_MASK);
 491 }
 492
 493
 494 static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)
 495 {
 496         if (spec->type == UVERBS_ATTR_TYPE_IDR ||
 497             spec->type == UVERBS_ATTR_TYPE_FD)
 498                 return spec->u.obj.obj_type;
 499         if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)
 500                 return spec->u2.objs_arr.obj_type;
 501         return UVERBS_API_KEY_ERR;
 502 }
 503
 504 static void uapi_key_okay(u32 key)
 505 {
 506         unsigned int count = 0;
 507
 508         if (uapi_key_is_object(key))
 509                 count++;
 510         if (uapi_key_is_ioctl_method(key))
 511                 count++;
 512         if (uapi_key_is_write_method(key))
 513                 count++;
 514         if (uapi_key_is_write_ex_method(key))
 515                 count++;
 516         if (uapi_key_is_attr(key))
 517                 count++;
 518         WARN(count != 1, "Bad count %d key=%x", count, key);
 519 }
 520
 521 static void uapi_finalize_disable(struct uverbs_api *uapi)
 522 {
 523         struct radix_tree_iter iter;
 524         u32 starting_key = 0;
 525         bool scan_again = false;
 526         void __rcu **slot;
 527
 528 again:
 529         radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {
 530                 uapi_key_okay(iter.index);
 531
 532                 if (uapi_key_is_object(iter.index)) {
 533                         struct uverbs_api_object *obj_elm =
 534                                 rcu_dereference_protected(*slot, true);
 535
 536                         if (obj_elm->disabled) {
 537                                 /* Have to check all the attrs again */
 538                                 scan_again = true;
 539                                 starting_key = iter.index;
 540                                 uapi_remove_object(uapi, iter.index);
 541                                 goto again;
 542                         }
 543                         continue;
 544                 }
 545
 546                 if (uapi_key_is_ioctl_method(iter.index)) {
 547                         struct uverbs_api_ioctl_method *method_elm =
 548                                 rcu_dereference_protected(*slot, true);
 549
 550                         if (method_elm->disabled) {
 551                                 starting_key = iter.index;
 552                                 uapi_remove_method(uapi, iter.index);
 553                                 goto again;
 554                         }
 555                         continue;
 556                 }
 557
 558                 if (uapi_key_is_write_method(iter.index) ||
 559                     uapi_key_is_write_ex_method(iter.index)) {
 560                         struct uverbs_api_write_method *method_elm =
 561                                 rcu_dereference_protected(*slot, true);
 562
 563                         if (method_elm->disabled) {
 564                                 kfree(method_elm);
 565                                 radix_tree_iter_delete(&uapi->radix, &iter, slot);
 566                         }
 567                         continue;
 568                 }
 569
 570                 if (uapi_key_is_attr(iter.index)) {
 571                         struct uverbs_api_attr *attr_elm =
 572                                 rcu_dereference_protected(*slot, true);
 573                         const struct uverbs_api_object *tmp_obj;
 574                         u32 obj_key;
 575
 576                         /*
 577                          * If the method has a mandatory object handle
 578                          * attribute which relies on an object which is not
 579                          * present then the entire method is uncallable.
 580                          */
 581                         if (!attr_elm->spec.mandatory)
 582                                 continue;
 583                         obj_key = uapi_get_obj_id(&attr_elm->spec);
 584                         if (obj_key == UVERBS_API_KEY_ERR)
 585                                 continue;
 586                         tmp_obj = uapi_get_object(uapi, obj_key);
 587                         if (tmp_obj && !tmp_obj->disabled)
 588                                 continue;
 589
 590                         starting_key = iter.index;
 591                         uapi_remove_method(
 592                                 uapi,
 593                                 iter.index & (UVERBS_API_OBJ_KEY_MASK |
 594                                               UVERBS_API_METHOD_KEY_MASK));
 595                         goto again;
 596                 }
 597
 598                 WARN_ON(false);
 599         }
 600
 601         if (!scan_again)
 602                 return;
 603         scan_again = false;
 604         starting_key = 0;
 605         goto again;
 606 }
 607
 608 void uverbs_destroy_api(struct uverbs_api *uapi)
 609 {
 610         if (!uapi)
 611                 return;
 612
 613         uapi_remove_range(uapi, 0, U32_MAX);
 614         kfree(uapi->write_methods);
 615         kfree(uapi);
 616 }
 617
 618 static const struct uapi_definition uverbs_core_api[] = {
 619         UAPI_DEF_CHAIN(uverbs_def_obj_counters),
 620         UAPI_DEF_CHAIN(uverbs_def_obj_cq),
 621         UAPI_DEF_CHAIN(uverbs_def_obj_dm),
 622         UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
 623         UAPI_DEF_CHAIN(uverbs_def_obj_intf),
 624         UAPI_DEF_CHAIN(uverbs_def_obj_mr),
 625         UAPI_DEF_CHAIN(uverbs_def_write_intf),
 626         {},
 627 };
 628
 629 struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
 630 {
 631         struct uverbs_api *uapi;
 632         int rc;
 633
 634         uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
 635         if (!uapi)
 636                 return ERR_PTR(-ENOMEM);
 637
 638         INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
 639         uapi->driver_id = ibdev->driver_id;
 640
 641         rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
 642         if (rc)
 643                 goto err;
 644         rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);
 645         if (rc)
 646                 goto err;
 647
 648         uapi_finalize_disable(uapi);
 649         rc = uapi_finalize(uapi);
 650         if (rc)
 651                 goto err;
 652
 653         return uapi;
 654 err:
 655         if (rc != -ENOMEM)
 656                 dev_err(&ibdev->dev,
 657                         "Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
 658                         rc);
 659
 660         uverbs_destroy_api(uapi);
 661         return ERR_PTR(rc);
 662 }
 663
 664 /*
 665  * The pre version is done before destroying the HW objects, it only blocks
 666  * off method access. All methods that require the ib_dev or the module data
 667  * must test one of these assignments prior to continuing.
 668  */
 669 void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
 670 {
 671         struct uverbs_api *uapi = uverbs_dev->uapi;
 672         struct radix_tree_iter iter;
 673         void __rcu **slot;
 674
 675         rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
 676
 677         radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
 678                 if (uapi_key_is_ioctl_method(iter.index)) {
 679                         struct uverbs_api_ioctl_method *method_elm =
 680                                 rcu_dereference_protected(*slot, true);
 681
 682                         if (method_elm->driver_method)
 683                                 rcu_assign_pointer(method_elm->handler, NULL);
 684                 }
 685         }
 686
 687         synchronize_srcu(&uverbs_dev->disassociate_srcu);
 688 }
 689
 690 /*
 691  * Called when a driver disassociates from the ib_uverbs_device. The
 692  * assumption is that the driver module will unload after. Replace everything
 693  * related to the driver with NULL as a safety measure.
 694  */
 695 void uverbs_disassociate_api(struct uverbs_api *uapi)
 696 {
 697         struct radix_tree_iter iter;
 698         void __rcu **slot;
 699
 700         radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
 701                 if (uapi_key_is_object(iter.index)) {
 702                         struct uverbs_api_object *object_elm =
 703                                 rcu_dereference_protected(*slot, true);
 704
 705                         /*
 706                          * Some type_attrs are in the driver module. We don't
 707                          * bother to keep track of which since there should be
 708                          * no use of this after disassociate.
 709                          */
 710                         object_elm->type_attrs = NULL;
 711                 } else if (uapi_key_is_attr(iter.index)) {
 712                         struct uverbs_api_attr *elm =
 713                                 rcu_dereference_protected(*slot, true);
 714
 715                         if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
 716                                 elm->spec.u2.enum_def.ids = NULL;
 717                 }
 718         }
 719 }