1
2
3
4
5#include <rdma/uverbs_ioctl.h>
6#include <rdma/rdma_user_ioctl.h>
7#include <linux/bitops.h>
8#include "rdma_core.h"
9#include "uverbs.h"
10
11static int ib_uverbs_notsupp(struct uverbs_attr_bundle *attrs)
12{
13 return -EOPNOTSUPP;
14}
15
16static void *uapi_add_elm(struct uverbs_api *uapi, u32 key, size_t alloc_size)
17{
18 void *elm;
19 int rc;
20
21 if (key == UVERBS_API_KEY_ERR)
22 return ERR_PTR(-EOVERFLOW);
23
24 elm = kzalloc(alloc_size, GFP_KERNEL);
25 rc = radix_tree_insert(&uapi->radix, key, elm);
26 if (rc) {
27 kfree(elm);
28 return ERR_PTR(rc);
29 }
30
31 return elm;
32}
33
34static void *uapi_add_get_elm(struct uverbs_api *uapi, u32 key,
35 size_t alloc_size, bool *exists)
36{
37 void *elm;
38
39 elm = uapi_add_elm(uapi, key, alloc_size);
40 if (!IS_ERR(elm)) {
41 *exists = false;
42 return elm;
43 }
44
45 if (elm != ERR_PTR(-EEXIST))
46 return elm;
47
48 elm = radix_tree_lookup(&uapi->radix, key);
49 if (WARN_ON(!elm))
50 return ERR_PTR(-EINVAL);
51 *exists = true;
52 return elm;
53}
54
55static int uapi_create_write(struct uverbs_api *uapi,
56 struct ib_device *ibdev,
57 const struct uapi_definition *def,
58 u32 obj_key,
59 u32 *cur_method_key)
60{
61 struct uverbs_api_write_method *method_elm;
62 u32 method_key = obj_key;
63 bool exists;
64
65 if (def->write.is_ex)
66 method_key |= uapi_key_write_ex_method(def->write.command_num);
67 else
68 method_key |= uapi_key_write_method(def->write.command_num);
69
70 method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
71 &exists);
72 if (IS_ERR(method_elm))
73 return PTR_ERR(method_elm);
74
75 if (WARN_ON(exists && (def->write.is_ex != method_elm->is_ex)))
76 return -EINVAL;
77
78 method_elm->is_ex = def->write.is_ex;
79 method_elm->handler = def->func_write;
80 if (def->write.is_ex)
81 method_elm->disabled = !(ibdev->uverbs_ex_cmd_mask &
82 BIT_ULL(def->write.command_num));
83 else
84 method_elm->disabled = !(ibdev->uverbs_cmd_mask &
85 BIT_ULL(def->write.command_num));
86
87 if (!def->write.is_ex && def->func_write) {
88 method_elm->has_udata = def->write.has_udata;
89 method_elm->has_resp = def->write.has_resp;
90 method_elm->req_size = def->write.req_size;
91 method_elm->resp_size = def->write.resp_size;
92 }
93
94 *cur_method_key = method_key;
95 return 0;
96}
97
98static int uapi_merge_method(struct uverbs_api *uapi,
99 struct uverbs_api_object *obj_elm, u32 obj_key,
100 const struct uverbs_method_def *method,
101 bool is_driver)
102{
103 u32 method_key = obj_key | uapi_key_ioctl_method(method->id);
104 struct uverbs_api_ioctl_method *method_elm;
105 unsigned int i;
106 bool exists;
107
108 if (!method->attrs)
109 return 0;
110
111 method_elm = uapi_add_get_elm(uapi, method_key, sizeof(*method_elm),
112 &exists);
113 if (IS_ERR(method_elm))
114 return PTR_ERR(method_elm);
115 if (exists) {
116
117
118
119 if (WARN_ON(method->handler))
120 return -EINVAL;
121 } else {
122 WARN_ON(!method->handler);
123 rcu_assign_pointer(method_elm->handler, method->handler);
124 if (method->handler != uverbs_destroy_def_handler)
125 method_elm->driver_method = is_driver;
126 }
127
128 for (i = 0; i != method->num_attrs; i++) {
129 const struct uverbs_attr_def *attr = (*method->attrs)[i];
130 struct uverbs_api_attr *attr_slot;
131
132 if (!attr)
133 continue;
134
135
136
137
138
139
140 if (attr->attr.type == UVERBS_ATTR_TYPE_ENUM_IN)
141 method_elm->driver_method |= is_driver;
142
143
144
145
146
147 if (attr->attr.type == UVERBS_ATTR_TYPE_IDRS_ARRAY) {
148 u8 access = attr->attr.u2.objs_arr.access;
149
150 if (WARN_ON(access == UVERBS_ACCESS_NEW ||
151 access == UVERBS_ACCESS_DESTROY))
152 return -EINVAL;
153 }
154
155 attr_slot =
156 uapi_add_elm(uapi, method_key | uapi_key_attr(attr->id),
157 sizeof(*attr_slot));
158
159 if (IS_ERR(attr_slot))
160 return PTR_ERR(attr_slot);
161
162 attr_slot->spec = attr->attr;
163 }
164
165 return 0;
166}
167
168static int uapi_merge_obj_tree(struct uverbs_api *uapi,
169 const struct uverbs_object_def *obj,
170 bool is_driver)
171{
172 struct uverbs_api_object *obj_elm;
173 unsigned int i;
174 u32 obj_key;
175 bool exists;
176 int rc;
177
178 obj_key = uapi_key_obj(obj->id);
179 obj_elm = uapi_add_get_elm(uapi, obj_key, sizeof(*obj_elm), &exists);
180 if (IS_ERR(obj_elm))
181 return PTR_ERR(obj_elm);
182
183 if (obj->type_attrs) {
184 if (WARN_ON(obj_elm->type_attrs))
185 return -EINVAL;
186
187 obj_elm->id = obj->id;
188 obj_elm->type_attrs = obj->type_attrs;
189 obj_elm->type_class = obj->type_attrs->type_class;
190
191
192
193
194
195
196
197
198
199
200 if (WARN_ON(is_driver &&
201 obj->type_attrs->type_class != &uverbs_idr_class &&
202 obj->type_attrs->type_class != &uverbs_fd_class))
203 return -EINVAL;
204 }
205
206 if (!obj->methods)
207 return 0;
208
209 for (i = 0; i != obj->num_methods; i++) {
210 const struct uverbs_method_def *method = (*obj->methods)[i];
211
212 if (!method)
213 continue;
214
215 rc = uapi_merge_method(uapi, obj_elm, obj_key, method,
216 is_driver);
217 if (rc)
218 return rc;
219 }
220
221 return 0;
222}
223
224static int uapi_disable_elm(struct uverbs_api *uapi,
225 const struct uapi_definition *def,
226 u32 obj_key,
227 u32 method_key)
228{
229 bool exists;
230
231 if (def->scope == UAPI_SCOPE_OBJECT) {
232 struct uverbs_api_object *obj_elm;
233
234 obj_elm = uapi_add_get_elm(
235 uapi, obj_key, sizeof(*obj_elm), &exists);
236 if (IS_ERR(obj_elm))
237 return PTR_ERR(obj_elm);
238 obj_elm->disabled = 1;
239 return 0;
240 }
241
242 if (def->scope == UAPI_SCOPE_METHOD &&
243 uapi_key_is_ioctl_method(method_key)) {
244 struct uverbs_api_ioctl_method *method_elm;
245
246 method_elm = uapi_add_get_elm(uapi, method_key,
247 sizeof(*method_elm), &exists);
248 if (IS_ERR(method_elm))
249 return PTR_ERR(method_elm);
250 method_elm->disabled = 1;
251 return 0;
252 }
253
254 if (def->scope == UAPI_SCOPE_METHOD &&
255 (uapi_key_is_write_method(method_key) ||
256 uapi_key_is_write_ex_method(method_key))) {
257 struct uverbs_api_write_method *write_elm;
258
259 write_elm = uapi_add_get_elm(uapi, method_key,
260 sizeof(*write_elm), &exists);
261 if (IS_ERR(write_elm))
262 return PTR_ERR(write_elm);
263 write_elm->disabled = 1;
264 return 0;
265 }
266
267 WARN_ON(true);
268 return -EINVAL;
269}
270
271static int uapi_merge_def(struct uverbs_api *uapi, struct ib_device *ibdev,
272 const struct uapi_definition *def_list,
273 bool is_driver)
274{
275 const struct uapi_definition *def = def_list;
276 u32 cur_obj_key = UVERBS_API_KEY_ERR;
277 u32 cur_method_key = UVERBS_API_KEY_ERR;
278 bool exists;
279 int rc;
280
281 if (!def_list)
282 return 0;
283
284 for (;; def++) {
285 switch ((enum uapi_definition_kind)def->kind) {
286 case UAPI_DEF_CHAIN:
287 rc = uapi_merge_def(uapi, ibdev, def->chain, is_driver);
288 if (rc)
289 return rc;
290 continue;
291
292 case UAPI_DEF_CHAIN_OBJ_TREE:
293 if (WARN_ON(def->object_start.object_id !=
294 def->chain_obj_tree->id))
295 return -EINVAL;
296
297 cur_obj_key = uapi_key_obj(def->object_start.object_id);
298 rc = uapi_merge_obj_tree(uapi, def->chain_obj_tree,
299 is_driver);
300 if (rc)
301 return rc;
302 continue;
303
304 case UAPI_DEF_END:
305 return 0;
306
307 case UAPI_DEF_IS_SUPPORTED_DEV_FN: {
308 void **ibdev_fn =
309 (void *)(&ibdev->ops) + def->needs_fn_offset;
310
311 if (*ibdev_fn)
312 continue;
313 rc = uapi_disable_elm(
314 uapi, def, cur_obj_key, cur_method_key);
315 if (rc)
316 return rc;
317 continue;
318 }
319
320 case UAPI_DEF_IS_SUPPORTED_FUNC:
321 if (def->func_is_supported(ibdev))
322 continue;
323 rc = uapi_disable_elm(
324 uapi, def, cur_obj_key, cur_method_key);
325 if (rc)
326 return rc;
327 continue;
328
329 case UAPI_DEF_OBJECT_START: {
330 struct uverbs_api_object *obj_elm;
331
332 cur_obj_key = uapi_key_obj(def->object_start.object_id);
333 obj_elm = uapi_add_get_elm(uapi, cur_obj_key,
334 sizeof(*obj_elm), &exists);
335 if (IS_ERR(obj_elm))
336 return PTR_ERR(obj_elm);
337 continue;
338 }
339
340 case UAPI_DEF_WRITE:
341 rc = uapi_create_write(
342 uapi, ibdev, def, cur_obj_key, &cur_method_key);
343 if (rc)
344 return rc;
345 continue;
346 }
347 WARN_ON(true);
348 return -EINVAL;
349 }
350}
351
352static int
353uapi_finalize_ioctl_method(struct uverbs_api *uapi,
354 struct uverbs_api_ioctl_method *method_elm,
355 u32 method_key)
356{
357 struct radix_tree_iter iter;
358 unsigned int num_attrs = 0;
359 unsigned int max_bkey = 0;
360 bool single_uobj = false;
361 void __rcu **slot;
362
363 method_elm->destroy_bkey = UVERBS_API_ATTR_BKEY_LEN;
364 radix_tree_for_each_slot (slot, &uapi->radix, &iter,
365 uapi_key_attrs_start(method_key)) {
366 struct uverbs_api_attr *elm =
367 rcu_dereference_protected(*slot, true);
368 u32 attr_key = iter.index & UVERBS_API_ATTR_KEY_MASK;
369 u32 attr_bkey = uapi_bkey_attr(attr_key);
370 u8 type = elm->spec.type;
371
372 if (uapi_key_attr_to_ioctl_method(iter.index) !=
373 uapi_key_attr_to_ioctl_method(method_key))
374 break;
375
376 if (elm->spec.mandatory)
377 __set_bit(attr_bkey, method_elm->attr_mandatory);
378
379 if (elm->spec.is_udata)
380 method_elm->has_udata = true;
381
382 if (type == UVERBS_ATTR_TYPE_IDR ||
383 type == UVERBS_ATTR_TYPE_FD) {
384 u8 access = elm->spec.u.obj.access;
385
386
387
388
389
390
391 if (access == UVERBS_ACCESS_NEW ||
392 access == UVERBS_ACCESS_DESTROY) {
393 if (WARN_ON(single_uobj))
394 return -EINVAL;
395
396 single_uobj = true;
397 if (WARN_ON(!elm->spec.mandatory))
398 return -EINVAL;
399 }
400
401 if (access == UVERBS_ACCESS_DESTROY)
402 method_elm->destroy_bkey = attr_bkey;
403 }
404
405 max_bkey = max(max_bkey, attr_bkey);
406 num_attrs++;
407 }
408
409 method_elm->key_bitmap_len = max_bkey + 1;
410 WARN_ON(method_elm->key_bitmap_len > UVERBS_API_ATTR_BKEY_LEN);
411
412 uapi_compute_bundle_size(method_elm, num_attrs);
413 return 0;
414}
415
416static int uapi_finalize(struct uverbs_api *uapi)
417{
418 const struct uverbs_api_write_method **data;
419 unsigned long max_write_ex = 0;
420 unsigned long max_write = 0;
421 struct radix_tree_iter iter;
422 void __rcu **slot;
423 int rc;
424 int i;
425
426 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
427 struct uverbs_api_ioctl_method *method_elm =
428 rcu_dereference_protected(*slot, true);
429
430 if (uapi_key_is_ioctl_method(iter.index)) {
431 rc = uapi_finalize_ioctl_method(uapi, method_elm,
432 iter.index);
433 if (rc)
434 return rc;
435 }
436
437 if (uapi_key_is_write_method(iter.index))
438 max_write = max(max_write,
439 iter.index & UVERBS_API_ATTR_KEY_MASK);
440 if (uapi_key_is_write_ex_method(iter.index))
441 max_write_ex =
442 max(max_write_ex,
443 iter.index & UVERBS_API_ATTR_KEY_MASK);
444 }
445
446 uapi->notsupp_method.handler = ib_uverbs_notsupp;
447 uapi->num_write = max_write + 1;
448 uapi->num_write_ex = max_write_ex + 1;
449 data = kmalloc_array(uapi->num_write + uapi->num_write_ex,
450 sizeof(*uapi->write_methods), GFP_KERNEL);
451 for (i = 0; i != uapi->num_write + uapi->num_write_ex; i++)
452 data[i] = &uapi->notsupp_method;
453 uapi->write_methods = data;
454 uapi->write_ex_methods = data + uapi->num_write;
455
456 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
457 if (uapi_key_is_write_method(iter.index))
458 uapi->write_methods[iter.index &
459 UVERBS_API_ATTR_KEY_MASK] =
460 rcu_dereference_protected(*slot, true);
461 if (uapi_key_is_write_ex_method(iter.index))
462 uapi->write_ex_methods[iter.index &
463 UVERBS_API_ATTR_KEY_MASK] =
464 rcu_dereference_protected(*slot, true);
465 }
466
467 return 0;
468}
469
470static void uapi_remove_range(struct uverbs_api *uapi, u32 start, u32 last)
471{
472 struct radix_tree_iter iter;
473 void __rcu **slot;
474
475 radix_tree_for_each_slot (slot, &uapi->radix, &iter, start) {
476 if (iter.index > last)
477 return;
478 kfree(rcu_dereference_protected(*slot, true));
479 radix_tree_iter_delete(&uapi->radix, &iter, slot);
480 }
481}
482
483static void uapi_remove_object(struct uverbs_api *uapi, u32 obj_key)
484{
485 uapi_remove_range(uapi, obj_key,
486 obj_key | UVERBS_API_METHOD_KEY_MASK |
487 UVERBS_API_ATTR_KEY_MASK);
488}
489
490static void uapi_remove_method(struct uverbs_api *uapi, u32 method_key)
491{
492 uapi_remove_range(uapi, method_key,
493 method_key | UVERBS_API_ATTR_KEY_MASK);
494}
495
496
497static u32 uapi_get_obj_id(struct uverbs_attr_spec *spec)
498{
499 if (spec->type == UVERBS_ATTR_TYPE_IDR ||
500 spec->type == UVERBS_ATTR_TYPE_FD)
501 return spec->u.obj.obj_type;
502 if (spec->type == UVERBS_ATTR_TYPE_IDRS_ARRAY)
503 return spec->u2.objs_arr.obj_type;
504 return UVERBS_API_KEY_ERR;
505}
506
507static void uapi_key_okay(u32 key)
508{
509 unsigned int count = 0;
510
511 if (uapi_key_is_object(key))
512 count++;
513 if (uapi_key_is_ioctl_method(key))
514 count++;
515 if (uapi_key_is_write_method(key))
516 count++;
517 if (uapi_key_is_write_ex_method(key))
518 count++;
519 if (uapi_key_is_attr(key))
520 count++;
521 WARN(count != 1, "Bad count %d key=%x", count, key);
522}
523
524static void uapi_finalize_disable(struct uverbs_api *uapi)
525{
526 struct radix_tree_iter iter;
527 u32 starting_key = 0;
528 bool scan_again = false;
529 void __rcu **slot;
530
531again:
532 radix_tree_for_each_slot (slot, &uapi->radix, &iter, starting_key) {
533 uapi_key_okay(iter.index);
534
535 if (uapi_key_is_object(iter.index)) {
536 struct uverbs_api_object *obj_elm =
537 rcu_dereference_protected(*slot, true);
538
539 if (obj_elm->disabled) {
540
541 scan_again = true;
542 starting_key = iter.index;
543 uapi_remove_object(uapi, iter.index);
544 goto again;
545 }
546 continue;
547 }
548
549 if (uapi_key_is_ioctl_method(iter.index)) {
550 struct uverbs_api_ioctl_method *method_elm =
551 rcu_dereference_protected(*slot, true);
552
553 if (method_elm->disabled) {
554 starting_key = iter.index;
555 uapi_remove_method(uapi, iter.index);
556 goto again;
557 }
558 continue;
559 }
560
561 if (uapi_key_is_write_method(iter.index) ||
562 uapi_key_is_write_ex_method(iter.index)) {
563 struct uverbs_api_write_method *method_elm =
564 rcu_dereference_protected(*slot, true);
565
566 if (method_elm->disabled) {
567 kfree(method_elm);
568 radix_tree_iter_delete(&uapi->radix, &iter, slot);
569 }
570 continue;
571 }
572
573 if (uapi_key_is_attr(iter.index)) {
574 struct uverbs_api_attr *attr_elm =
575 rcu_dereference_protected(*slot, true);
576 const struct uverbs_api_object *tmp_obj;
577 u32 obj_key;
578
579
580
581
582
583
584 if (!attr_elm->spec.mandatory)
585 continue;
586 obj_key = uapi_get_obj_id(&attr_elm->spec);
587 if (obj_key == UVERBS_API_KEY_ERR)
588 continue;
589 tmp_obj = uapi_get_object(uapi, obj_key);
590 if (IS_ERR(tmp_obj)) {
591 if (PTR_ERR(tmp_obj) == -ENOMSG)
592 continue;
593 } else {
594 if (!tmp_obj->disabled)
595 continue;
596 }
597
598 starting_key = iter.index;
599 uapi_remove_method(
600 uapi,
601 iter.index & (UVERBS_API_OBJ_KEY_MASK |
602 UVERBS_API_METHOD_KEY_MASK));
603 goto again;
604 }
605
606 WARN_ON(false);
607 }
608
609 if (!scan_again)
610 return;
611 scan_again = false;
612 starting_key = 0;
613 goto again;
614}
615
616void uverbs_destroy_api(struct uverbs_api *uapi)
617{
618 if (!uapi)
619 return;
620
621 uapi_remove_range(uapi, 0, U32_MAX);
622 kfree(uapi->write_methods);
623 kfree(uapi);
624}
625
626static const struct uapi_definition uverbs_core_api[] = {
627 UAPI_DEF_CHAIN(uverbs_def_obj_counters),
628 UAPI_DEF_CHAIN(uverbs_def_obj_cq),
629 UAPI_DEF_CHAIN(uverbs_def_obj_device),
630 UAPI_DEF_CHAIN(uverbs_def_obj_dm),
631 UAPI_DEF_CHAIN(uverbs_def_obj_flow_action),
632 UAPI_DEF_CHAIN(uverbs_def_obj_intf),
633 UAPI_DEF_CHAIN(uverbs_def_obj_mr),
634 UAPI_DEF_CHAIN(uverbs_def_write_intf),
635 {},
636};
637
638struct uverbs_api *uverbs_alloc_api(struct ib_device *ibdev)
639{
640 struct uverbs_api *uapi;
641 int rc;
642
643 uapi = kzalloc(sizeof(*uapi), GFP_KERNEL);
644 if (!uapi)
645 return ERR_PTR(-ENOMEM);
646
647 INIT_RADIX_TREE(&uapi->radix, GFP_KERNEL);
648 uapi->driver_id = ibdev->driver_id;
649
650 rc = uapi_merge_def(uapi, ibdev, uverbs_core_api, false);
651 if (rc)
652 goto err;
653 rc = uapi_merge_def(uapi, ibdev, ibdev->driver_def, true);
654 if (rc)
655 goto err;
656
657 uapi_finalize_disable(uapi);
658 rc = uapi_finalize(uapi);
659 if (rc)
660 goto err;
661
662 return uapi;
663err:
664 if (rc != -ENOMEM)
665 dev_err(&ibdev->dev,
666 "Setup of uverbs_api failed, kernel parsing tree description is not valid (%d)??\n",
667 rc);
668
669 uverbs_destroy_api(uapi);
670 return ERR_PTR(rc);
671}
672
673
674
675
676
677
678void uverbs_disassociate_api_pre(struct ib_uverbs_device *uverbs_dev)
679{
680 struct uverbs_api *uapi = uverbs_dev->uapi;
681 struct radix_tree_iter iter;
682 void __rcu **slot;
683
684 rcu_assign_pointer(uverbs_dev->ib_dev, NULL);
685
686 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
687 if (uapi_key_is_ioctl_method(iter.index)) {
688 struct uverbs_api_ioctl_method *method_elm =
689 rcu_dereference_protected(*slot, true);
690
691 if (method_elm->driver_method)
692 rcu_assign_pointer(method_elm->handler, NULL);
693 }
694 }
695
696 synchronize_srcu(&uverbs_dev->disassociate_srcu);
697}
698
699
700
701
702
703
704void uverbs_disassociate_api(struct uverbs_api *uapi)
705{
706 struct radix_tree_iter iter;
707 void __rcu **slot;
708
709 radix_tree_for_each_slot (slot, &uapi->radix, &iter, 0) {
710 if (uapi_key_is_object(iter.index)) {
711 struct uverbs_api_object *object_elm =
712 rcu_dereference_protected(*slot, true);
713
714
715
716
717
718
719 object_elm->type_attrs = NULL;
720 } else if (uapi_key_is_attr(iter.index)) {
721 struct uverbs_api_attr *elm =
722 rcu_dereference_protected(*slot, true);
723
724 if (elm->spec.type == UVERBS_ATTR_TYPE_ENUM_IN)
725 elm->spec.u2.enum_def.ids = NULL;
726 }
727 }
728}
729
