1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
3 * Copyright (c) 2017, Mellanox Technologies inc. All rights reserved.
5 #include <rdma/uverbs_ioctl.h>
6 #include <rdma/rdma_user_ioctl.h>
7 #include <linux/bitops.h>
11 static ssize_t ib_uverbs_notsupp(struct ib_uverbs_file *file,
12 const char __user *buf, int in_len,
18 static int ib_uverbs_ex_notsupp(struct ib_uverbs_file *file,
19 struct ib_udata *ucore, struct ib_udata *uhw)
24 static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
29 if (key == UVERBS_API_KEY_ERR)
30 return ERR_PTR(-EOVERFLOW);
32 elm = kzalloc(alloc_size, GFP_KERNEL);
33 rc = radix_tree_insert(&uapi->radix, key, elm);
42 static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,
43 size_t alloc_size, bool *exists)
47 elm = uapi_add_elm(uapi, key, alloc_size);
53 if (elm != ERR_PTR(-EEXIST))
56 elm = radix_tree_lookup(&uapi->radix, key);
58 return ERR_PTR(-EINVAL);
63 static int uapi_create_write(struct uverbs_api *uapi,
64 struct ib_device *ibdev,
65 const struct uapi_definition *def,
69 struct uverbs_api_write_method *method_elm;
70 u32 method_key = obj_key;
74 method_key |= uapi_key_write_ex_method(def->write.command_num);
76 method_key |= uapi_key_write_method(def->write.command_num);
78 method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
80 if (IS_ERR(method_elm))
81 return PTR_ERR(method_elm);
83 if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex ||
84 method_elm->handler_ex || method_elm->handler)))
87 method_elm->is_ex = def->write.is_ex;
88 if (def->write.is_ex) {
89 method_elm->handler_ex = def->func_write_ex;
91 method_elm->disabled = !(ibdev->uverbs_ex_cmd_mask &
92 BIT_ULL(def->write.command_num));
94 method_elm->handler = def->func_write;
96 method_elm->disabled = !(ibdev->uverbs_cmd_mask &
97 BIT_ULL(def->write.command_num));
100 *cur_method_key = method_key;
104 static int uapi_merge_method(struct uverbs_api *uapi,
105 struct uverbs_api_object *obj_elm, u32 obj_key,
106 const struct uverbs_method_def *method,
109 u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
110 struct uverbs_api_ioctl_method *method_elm;
117 method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
119 if (IS_ERR(method_elm))
120 return PTR_ERR(method_elm);
123 * This occurs when a driver uses ADD_UVERBS_ATTRIBUTES_SIMPLE
125 if (WARN_ON(method->handler))
128 WARN_ON(!method->handler);
129 rcu_assign_pointer(method_elm->handler, method->handler);
130 if (method->handler != uverbs_destroy_def_handler)
131 method_elm->driver_method = is_driver;
134 for (i = 0; i != method->num_attrs; i++) {
135 const struct uverbs_attr_def *attr = (*method->attrs)[i];
136 struct uverbs_api_attr *attr_slot;
142 * ENUM_IN contains the 'ids' pointer to the driver's .rodata,
143 * so if it is specified by a driver then it always makes this
144 * into a driver method.
146 if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
147 method_elm->driver_method |= is_driver;
150 * Like other uobject based things we only support a single
151 * uobject being NEW'd or DESTROY'd
153 if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
154 u8 access = attr->attr.u2.objs_arr.access;
156 if (WARN_ON(access == UVERBS_ACCESS_NEW ||
157 access == UVERBS_ACCESS_DESTROY))
162 uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
164 /* Attributes are not allowed to be modified by drivers */
165 if (IS_ERR(attr_slot))
166 return PTR_ERR(attr_slot);
168 attr_slot->spec = attr->attr;
174 static int uapi_merge_obj_tree(struct uverbs_api *uapi,
175 const struct uverbs_object_def *obj,
178 struct uverbs_api_object *obj_elm;
184 obj_key = uapi_key_obj(obj->id);
185 obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);
187 return PTR_ERR(obj_elm);
189 if (obj->type_attrs) {
190 if (WARN_ON(obj_elm->type_attrs))
193 obj_elm->type_attrs = obj->type_attrs;
194 obj_elm->type_class = obj->type_attrs->type_class;
196 * Today drivers are only permitted to use idr_class
197 * types. They cannot use FD types because we currently have
198 * no way to revoke the fops pointer after device
201 if (WARN_ON(is_driver &&
202 obj->type_attrs->type_class != &uverbs_idr_class))
209 for (i = 0; i != obj->num_methods; i++) {
210 const struct uverbs_method_def *method = (*obj->methods)[i];
215 rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
224 static int uapi_disable_elm(struct uverbs_api *uapi,
225 const struct uapi_definition *def,
231 if (def->scope == UAPI_SCOPE_OBJECT) {
232 struct uverbs_api_object *obj_elm;
234 obj_elm = uapi_add_get_elm(
235 uapi, obj_key, sizeof(*obj_elm), &exists);
237 return PTR_ERR(obj_elm);
238 obj_elm->disabled = 1;
242 if (def->scope == UAPI_SCOPE_METHOD &&
243 uapi_key_is_ioctl_method(method_key)) {
244 struct uverbs_api_ioctl_method *method_elm;
246 method_elm = uapi_add_get_elm(uapi, method_key,
247 sizeof(*method_elm), &exists);
248 if (IS_ERR(method_elm))
249 return PTR_ERR(method_elm);
250 method_elm->disabled = 1;
254 if (def->scope == UAPI_SCOPE_METHOD &&
255 (uapi_key_is_write_method(method_key) ||
256 uapi_key_is_write_ex_method(method_key))) {
257 struct uverbs_api_write_method *write_elm;
259 write_elm = uapi_add_get_elm(uapi, method_key,
260 sizeof(*write_elm), &exists);
261 if (IS_ERR(write_elm))
262 return PTR_ERR(write_elm);
263 write_elm->disabled = 1;
271 static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,
272 const struct uapi_definition *def_list,
275 const struct uapi_definition *def = def_list;
276 u32 cur_obj_key = UVERBS_API_KEY_ERR;
277 u32 cur_method_key = UVERBS_API_KEY_ERR;
285 switch ((enum uapi_definition_kind)def->kind) {
287 rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);
292 case UAPI_DEF_CHAIN_OBJ_TREE:
293 if (WARN_ON(def->object_start.object_id !=
294 def->chain_obj_tree->id))
297 cur_obj_key = uapi_key_obj(def->object_start.object_id);
298 rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,
307 case UAPI_DEF_IS_SUPPORTED_DEV_FN: {
308 void **ibdev_fn = (void *)ibdev + def->needs_fn_offset;
312 rc = uapi_disable_elm(
313 uapi, def, cur_obj_key, cur_method_key);
319 case UAPI_DEF_IS_SUPPORTED_FUNC:
320 if (def->func_is_supported(ibdev))
322 rc = uapi_disable_elm(
323 uapi, def, cur_obj_key, cur_method_key);
328 case UAPI_DEF_OBJECT_START: {
329 struct uverbs_api_object *obj_elm;
331 cur_obj_key = uapi_key_obj(def->object_start.object_id);
332 obj_elm = uapi_add_get_elm(uapi, cur_obj_key,
333 sizeof(*obj_elm), &exists);
335 return PTR_ERR(obj_elm);
340 rc = uapi_create_write(
341 uapi, ibdev, def, cur_obj_key, &cur_method_key);
352 uapi_finalize_ioctl_method(struct uverbs_api *uapi,
353 struct uverbs_api_ioctl_method *method_elm,
356 struct radix_tree_iter iter;
357 unsigned int num_attrs = 0;
358 unsigned int max_bkey = 0;
359 bool single_uobj = false;
362 method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
363 radix_tree_for_each_slot (slot, &uapi->radix, &iter,
364 uapi_key_attrs_start(method_key)) {
365 struct uverbs_api_attr *elm =
366 rcu_dereference_protected(*slot, true);
367 u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
368 u32 attr_bkey = uapi_bkey_attr(attr_key);
369 u8 type = elm->spec.type;
371 if (uapi_key_attr_to_ioctl_method(iter.index) !=
372 uapi_key_attr_to_ioctl_method(method_key))
375 if (elm->spec.mandatory)
376 __set_bit(attr_bkey, method_elm->attr_mandatory);
378 if (type == UVERBS_ATTR_TYPE_IDR ||
379 type == UVERBS_ATTR_TYPE_FD) {
380 u8 access = elm->spec.u.obj.access;
383 * Verbs specs may only have one NEW/DESTROY, we don't
384 * have the infrastructure to abort multiple NEW's or
385 * cope with multiple DESTROY failure.
387 if (access == UVERBS_ACCESS_NEW ||
388 access == UVERBS_ACCESS_DESTROY) {
389 if (WARN_ON(single_uobj))
393 if (WARN_ON(!elm->spec.mandatory))
397 if (access == UVERBS_ACCESS_DESTROY)
398 method_elm->destroy_bkey = attr_bkey;
401 max_bkey = max(max_bkey, attr_bkey);
405 method_elm->key_bitmap_len = max_bkey + 1;
406 WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);
408 uapi_compute_bundle_size(method_elm, num_attrs);
412 static int uapi_finalize(struct uverbs_api *uapi)
414 const struct uverbs_api_write_method **data;
415 unsigned long max_write_ex = 0;
416 unsigned long max_write = 0;
417 struct radix_tree_iter iter;
422 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
423 struct uverbs_api_ioctl_method *method_elm =
424 rcu_dereference_protected(*slot, true);
426 if (uapi_key_is_ioctl_method(iter.index)) {
427 rc = uapi_finalize_ioctl_method(uapi, method_elm,
433 if (uapi_key_is_write_method(iter.index))
434 max_write = max(max_write,
435 iter.index & UVERBS_API_ATTR_KEY_MASK);
436 if (uapi_key_is_write_ex_method(iter.index))
439 iter.index & UVERBS_API_ATTR_KEY_MASK);
442 uapi->notsupp_method.handler = ib_uverbs_notsupp;
443 uapi->notsupp_method.handler_ex = ib_uverbs_ex_notsupp;
444 uapi->num_write = max_write + 1;
445 uapi->num_write_ex = max_write_ex + 1;
446 data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
447 sizeof(*uapi->write_methods), GFP_KERNEL);
448 for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
449 data[i] = &uapi->notsupp_method;
450 uapi->write_methods = data;
451 uapi->write_ex_methods = data + uapi->num_write;
453 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
454 if (uapi_key_is_write_method(iter.index))
455 uapi->write_methods[iter.index &
456 UVERBS_API_ATTR_KEY_MASK] =
457 rcu_dereference_protected(*slot, true);
458 if (uapi_key_is_write_ex_method(iter.index))
459 uapi->write_ex_methods[iter.index &
460 UVERBS_API_ATTR_KEY_MASK] =
461 rcu_dereference_protected(*slot, true);
467 static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)
469 struct radix_tree_iter iter;
472 radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {
473 if (iter.index > last)
475 kfree(rcu_dereference_protected(*slot, true));
476 radix_tree_iter_delete(&uapi->radix, &iter, slot);
480 static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)
482 uapi_remove_range(uapi, obj_key,
483 obj_key | UVERBS_API_METHOD_KEY_MASK |
484 UVERBS_API_ATTR_KEY_MASK);
487 static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)
489 uapi_remove_range(uapi, method_key,
490 method_key | UVERBS_API_ATTR_KEY_MASK);
494 static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)
496 if (spec->type == UVERBS_ATTR_TYPE_IDR ||
497 spec->type == UVERBS_ATTR_TYPE_FD)
498 return spec->u.obj.obj_type;
499 if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)
500 return spec->u2.objs_arr.obj_type;
501 return UVERBS_API_KEY_ERR;
504 static void uapi_key_okay(u32 key)
506 unsigned int count = 0;
508 if (uapi_key_is_object(key))
510 if (uapi_key_is_ioctl_method(key))
512 if (uapi_key_is_write_method(key))
514 if (uapi_key_is_write_ex_method(key))
516 if (uapi_key_is_attr(key))
518 WARN(count != 1, "Bad count %d key=%x", count, key);
521 static void uapi_finalize_disable(struct uverbs_api *uapi)
523 struct radix_tree_iter iter;
524 u32 starting_key = 0;
525 bool scan_again = false;
529 radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {
530 uapi_key_okay(iter.index);
532 if (uapi_key_is_object(iter.index)) {
533 struct uverbs_api_object *obj_elm =
534 rcu_dereference_protected(*slot, true);
536 if (obj_elm->disabled) {
537 /* Have to check all the attrs again */
539 starting_key = iter.index;
540 uapi_remove_object(uapi, iter.index);
546 if (uapi_key_is_ioctl_method(iter.index)) {
547 struct uverbs_api_ioctl_method *method_elm =
548 rcu_dereference_protected(*slot, true);
550 if (method_elm->disabled) {
551 starting_key = iter.index;
552 uapi_remove_method(uapi, iter.index);
558 if (uapi_key_is_write_method(iter.index) ||
559 uapi_key_is_write_ex_method(iter.index)) {
560 struct uverbs_api_write_method *method_elm =
561 rcu_dereference_protected(*slot, true);
563 if (method_elm->disabled) {
565 radix_tree_iter_delete(&uapi->radix, &iter, slot);
570 if (uapi_key_is_attr(iter.index)) {
571 struct uverbs_api_attr *attr_elm =
572 rcu_dereference_protected(*slot, true);
573 const struct uverbs_api_object *tmp_obj;
577 * If the method has a mandatory object handle
578 * attribute which relies on an object which is not
579 * present then the entire method is uncallable.
581 if (!attr_elm->spec.mandatory)
583 obj_key = uapi_get_obj_id(&attr_elm->spec);
584 if (obj_key == UVERBS_API_KEY_ERR)
586 tmp_obj = uapi_get_object(uapi, obj_key);
587 if (tmp_obj && !tmp_obj->disabled)
590 starting_key = iter.index;
593 iter.index & (UVERBS_API_OBJ_KEY_MASK |
594 UVERBS_API_METHOD_KEY_MASK));
608 void uverbs_destroy_api(struct uverbs_api *uapi)
613 uapi_remove_range(uapi, 0, U32_MAX);
614 kfree(uapi->write_methods);
618 static const struct uapi_definition uverbs_core_api[] = {
619 UAPI_DEF_CHAIN(uverbs_def_obj_counters),
620 UAPI_DEF_CHAIN(uverbs_def_obj_cq),
621 UAPI_DEF_CHAIN(uverbs_def_obj_dm),
622 UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
623 UAPI_DEF_CHAIN(uverbs_def_obj_intf),
624 UAPI_DEF_CHAIN(uverbs_def_obj_mr),
625 UAPI_DEF_CHAIN(uverbs_def_write_intf),
629 struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
631 struct uverbs_api *uapi;
634 uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
636 return ERR_PTR(-ENOMEM);
638 INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
639 uapi->driver_id = ibdev->driver_id;
641 rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
644 rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);
648 uapi_finalize_disable(uapi);
649 rc = uapi_finalize(uapi);
657 "Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
660 uverbs_destroy_api(uapi);
665 * The pre version is done before destroying the HW objects, it only blocks
666 * off method access. All methods that require the ib_dev or the module data
667 * must test one of these assignments prior to continuing.
669 void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
671 struct uverbs_api *uapi = uverbs_dev->uapi;
672 struct radix_tree_iter iter;
675 rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
677 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
678 if (uapi_key_is_ioctl_method(iter.index)) {
679 struct uverbs_api_ioctl_method *method_elm =
680 rcu_dereference_protected(*slot, true);
682 if (method_elm->driver_method)
683 rcu_assign_pointer(method_elm->handler, NULL);
687 synchronize_srcu(&uverbs_dev->disassociate_srcu);
691 * Called when a driver disassociates from the ib_uverbs_device. The
692 * assumption is that the driver module will unload after. Replace everything
693 * related to the driver with NULL as a safety measure.
695 void uverbs_disassociate_api(struct uverbs_api *uapi)
697 struct radix_tree_iter iter;
700 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
701 if (uapi_key_is_object(iter.index)) {
702 struct uverbs_api_object *object_elm =
703 rcu_dereference_protected(*slot, true);
706 * Some type_attrs are in the driver module. We don't
707 * bother to keep track of which since there should be
708 * no use of this after disassociate.
710 object_elm->type_attrs = NULL;
711 } else if (uapi_key_is_attr(iter.index)) {
712 struct uverbs_api_attr *elm =
713 rcu_dereference_protected(*slot, true);
715 if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
716 elm->spec.u2.enum_def.ids = NULL;