1
2
3
4
5
6
7
8#include "ai-engine-internal.h"
9#include <linux/slab.h>
10
11
12
13
14#define AIE_RSC_BITMAP_TILETYPE_BITSHIFT 0U
15#define AIE_RSC_BITMAP_TILETYPE_BITWIDTH 4U
16#define AIE_RSC_BITMAP_MODTYPE_BITSHIFT 4U
17#define AIE_RSC_BITMAP_MODTYPE_BITWIDTH 4U
18#define AIE_RSC_BITMAP_RSCTYPE_BITSHIFT 8U
19#define AIE_RSC_BITMAP_RSCTYPE_BITWIDTH 8U
20#define AIE_RSC_BITMAP_LENU64_BITSHIFT 16U
21#define AIE_RSC_BITMAP_LENU64_BITWIDTH 32U
22
23#define AIE_RSC_BITMAP_GEN_MASK(N) \
24 GENMASK_ULL((AIE_RSC_BITMAP_##N ##_BITSHIFT + \
25 AIE_RSC_BITMAP_##N ##_BITWIDTH - 1), \
26 AIE_RSC_BITMAP_##N ##_BITSHIFT)
27#define AIE_RSC_BITMAP_TILETYPE_MASK AIE_RSC_BITMAP_GEN_MASK(TILETYPE)
28#define AIE_RSC_BITMAP_MODTYPE_MASK AIE_RSC_BITMAP_GEN_MASK(MODTYPE)
29#define AIE_RSC_BITMAP_RSCTYPE_MASK AIE_RSC_BITMAP_GEN_MASK(RSCTYPE)
30#define AIE_RSC_BITMAP_LENU64_MASK AIE_RSC_BITMAP_GEN_MASK(LENU64)
31
32#define AIE_RSC_BITMAP_HEAD_VAL(N, v) \
33 (((v) & AIE_RSC_BITMAP_##N ##_MASK) >> AIE_RSC_BITMAP_##N ##_BITSHIFT)
34
35
36
37
38enum aie_rsc_alloc_type {
39 AIE_RSC_ALLOC_STATIC = 0,
40 AIE_RSC_ALLOC_AVAIL = 1,
41 AIE_RSC_ALLOC_MAX = 2
42};
43
44
45
46
47
48
49struct aie_rsc_meta_header {
50 u64 stat;
51 u64 bitmap_off;
52};
53
54
55
56
57
58
59
60
61struct aie_rsc_bitmap {
62 u64 header;
63 u64 bitmap[0];
64};
65
66
67
68
69
70
71
72static inline
73struct aie_tile_attr *aie_dev_get_tile_attr(struct aie_device *adev,
74 enum aie_tile_type ttype)
75{
76 return &adev->ttype_attr[ttype];
77}
78
79
80
81
82
83
84
85
86
87static inline const
88struct aie_tile_rsc_attr *aie_dev_get_tile_rsc_attr(struct aie_device *adev,
89 enum aie_tile_type ttype,
90 enum aie_rsc_type rtype)
91{
92 return &adev->ttype_attr[ttype].rscs_attr[rtype];
93}
94
95
96
97
98
99
100
101
102
103
104
105
106static int aie_dev_get_mod_id(struct aie_device *adev,
107 enum aie_tile_type ttype,
108 enum aie_module_type mod)
109{
110 struct aie_tile_attr *tattr = &adev->ttype_attr[ttype];
111
112 int ret;
113
114 if (ttype == AIE_TILE_TYPE_TILE)
115 ret = AIE_MOD_ID(TILE, mod);
116 else if (ttype == AIE_TILE_TYPE_SHIMPL)
117 ret = AIE_MOD_ID(SHIMPL, mod);
118 else
119 ret = AIE_MOD_ID(SHIMNOC, mod);
120
121 if (ret < 0 || ret > tattr->num_mods)
122 return -EINVAL;
123
124 return ret;
125}
126
127
128
129
130
131
132
133
134
135
136static const
137struct aie_mod_rsc_attr *aie_dev_get_mod_rsc_attr(struct aie_device *adev,
138 enum aie_tile_type ttype,
139 enum aie_module_type mod,
140 enum aie_rsc_type rtype)
141{
142 const struct aie_tile_rsc_attr *rsc = aie_dev_get_tile_rsc_attr(adev,
143 ttype,
144 rtype);
145 const struct aie_mod_rsc_attr *mrsc = NULL;
146 int mod_id = aie_dev_get_mod_id(adev, ttype, mod);
147
148 if (mod_id < 0)
149 return NULL;
150
151 mrsc = &rsc->mod_attr[mod_id];
152 if (mrsc && !mrsc->num_rscs)
153 return NULL;
154
155 return mrsc;
156}
157
158
159
160
161
162
163
164
165
166
167
168
169
170static
171struct aie_rsc_stat *aie_part_get_ttype_rsc_bitmaps(struct aie_partition *apart,
172 enum aie_tile_type ttype,
173 enum aie_module_type mod,
174 enum aie_rsc_type rtype)
175{
176 int mod_id;
177 struct aie_mod_rscs *mrscs;
178
179 if (ttype >= AIE_TILE_TYPE_MAX)
180 return NULL;
181
182 mod_id = aie_dev_get_mod_id(apart->adev, ttype, mod);
183 if (mod_id < 0)
184 return NULL;
185
186 if (rtype >= AIE_RSCTYPE_MAX)
187 return NULL;
188
189 mrscs = apart->trscs[ttype].mod_rscs[rtype];
190 if (!mrscs)
191 return NULL;
192
193 return mrscs[mod_id].rscs_stat;
194}
195
196
197
198
199
200
201
202
203
204
205
206static
207struct aie_rsc_stat *aie_part_get_rsc_bitmaps(struct aie_partition *apart,
208 struct aie_location loc,
209 enum aie_module_type mod,
210 enum aie_rsc_type rtype)
211{
212 u32 ttype = apart->adev->ops->get_tile_type(&loc);
213
214 return aie_part_get_ttype_rsc_bitmaps(apart, ttype, mod, rtype);
215}
216
217
218
219
220
221
222
223
224
225
226
227static
228int aie_part_get_mod_num_rscs(struct aie_partition *apart,
229 struct aie_location loc,
230 enum aie_module_type mod,
231 enum aie_rsc_type rtype)
232{
233 u32 ttype = apart->adev->ops->get_tile_type(&loc);
234 const struct aie_mod_rsc_attr *mattr;
235
236 mattr = aie_dev_get_mod_rsc_attr(apart->adev, ttype, mod, rtype);
237 if (!mattr)
238 return 0;
239
240 return mattr->num_rscs;
241}
242
243
244
245
246
247
248
249
250
251
252
253
254
255static
256int aie_part_get_rsc_startbit(struct aie_partition *apart,
257 struct aie_location loc,
258 enum aie_module_type mod,
259 enum aie_rsc_type rtype)
260{
261 struct aie_device *adev = apart->adev;
262 u32 ttype;
263 const struct aie_mod_rsc_attr *mattr;
264 int num_rows;
265 struct aie_tile_attr *tattr;
266
267 ttype = adev->ops->get_tile_type(&loc);
268
269 mattr = aie_dev_get_mod_rsc_attr(adev, ttype, mod, rtype);
270 if (!mattr)
271 return -EINVAL;
272
273 num_rows = aie_part_get_tile_rows(apart, ttype);
274 tattr = &adev->ttype_attr[ttype];
275 return mattr->num_rscs *
276 ((loc.col - apart->range.start.col) * num_rows +
277 loc.row - tattr->start_row);
278}
279
280
281
282
283
284
285
286
287
288static
289int aie_part_adjust_loc(struct aie_partition *apart,
290 struct aie_location rloc, struct aie_location *loc)
291{
292 loc->col = rloc.col + apart->range.start.col;
293 loc->row = rloc.row + apart->range.start.row;
294
295 if (aie_validate_location(apart, *loc) < 0) {
296 dev_err(&apart->dev,
297 "invalid loc (%u,%u) in (%u,%u).\n",
298 rloc.col, rloc.row,
299 apart->range.size.col, apart->range.size.row);
300 return -EINVAL;
301 }
302
303 return 0;
304}
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322int aie_part_rscmgr_init(struct aie_partition *apart)
323{
324 struct aie_device *adev = apart->adev;
325 u32 t;
326
327 for (t = AIE_TILE_TYPE_TILE; t < AIE_TILE_TYPE_MAX; t++) {
328 struct aie_tile_rscs *trscs = &apart->trscs[t];
329 struct aie_tile_attr *tattr;
330 int num_rows, num_cols;
331 u32 r;
332
333
334
335
336
337
338
339 if (t == AIE_TILE_TYPE_SHIMNOC) {
340 *trscs = apart->trscs[AIE_TILE_TYPE_SHIMPL];
341 continue;
342 }
343
344
345
346
347
348
349 tattr = aie_dev_get_tile_attr(adev, t);
350 num_rows = aie_part_get_tile_rows(apart, t);
351 num_cols = apart->range.size.col;
352
353 for (r = AIE_RSCTYPE_PERF; r < AIE_RSCTYPE_MAX; r++) {
354 const struct aie_tile_rsc_attr *trsc_attr;
355 struct aie_mod_rscs *mod_rscs;
356 u32 m;
357
358 trsc_attr = aie_dev_get_tile_rsc_attr(adev, t, r);
359 if (!trsc_attr)
360 continue;
361
362 mod_rscs = kcalloc(tattr->num_mods,
363 sizeof(*mod_rscs), GFP_KERNEL);
364 if (!mod_rscs) {
365 aie_part_rscmgr_finish(apart);
366 return -ENOMEM;
367 }
368
369 trscs->mod_rscs[r] = mod_rscs;
370 for (m = 0 ; m < tattr->num_mods; m++) {
371 struct aie_rsc_stat *rscs_stat;
372 int num_mrscs = trsc_attr->mod_attr[m].num_rscs;
373 int ret, total_rscs;
374
375
376
377
378
379
380 if (!num_mrscs)
381 continue;
382
383 rscs_stat = kzalloc(sizeof(*rscs_stat),
384 GFP_KERNEL);
385 if (!rscs_stat) {
386 aie_part_rscmgr_finish(apart);
387 return -ENOMEM;
388 }
389
390 mod_rscs[m].rscs_stat = rscs_stat;
391 total_rscs = num_mrscs * num_rows * num_cols;
392
393
394
395
396 ret = aie_resource_initialize(&rscs_stat->rbits,
397 total_rscs);
398 if (ret) {
399 aie_part_rscmgr_finish(apart);
400 return ret;
401 }
402 ret = aie_resource_initialize(&rscs_stat->sbits,
403 total_rscs);
404 if (ret) {
405 aie_part_rscmgr_finish(apart);
406 return ret;
407 }
408 }
409 }
410 }
411
412
413 return aie_part_set_intr_rscs(apart);
414}
415
416
417
418
419
420
421void aie_part_rscmgr_finish(struct aie_partition *apart)
422{
423 struct aie_device *adev = apart->adev;
424 u32 t;
425
426 for (t = AIE_TILE_TYPE_TILE; t < AIE_TILE_TYPE_MAX; t++) {
427 struct aie_tile_rscs *trscs = &apart->trscs[t];
428 struct aie_tile_attr *tattr;
429 u32 r;
430
431
432 if (t == AIE_TILE_TYPE_SHIMNOC)
433 continue;
434
435 tattr = aie_dev_get_tile_attr(adev, t);
436 for (r = AIE_RSCTYPE_PERF; r < AIE_RSCTYPE_MAX; r++) {
437 struct aie_mod_rscs *mod_rscs;
438 u32 m;
439
440 mod_rscs = trscs->mod_rscs[r];
441 if (!mod_rscs)
442 continue;
443
444 for (m = 0 ; m < tattr->num_mods; m++) {
445 struct aie_rsc_stat *rscs_stat;
446
447 rscs_stat = mod_rscs[m].rscs_stat;
448 if (!rscs_stat)
449 continue;
450
451 aie_resource_uninitialize(&rscs_stat->rbits);
452 aie_resource_uninitialize(&rscs_stat->sbits);
453 }
454
455 kfree(mod_rscs);
456 trscs->mod_rscs[r] = NULL;
457 }
458 }
459}
460
461
462
463
464
465
466
467
468
469void aie_part_rscmgr_reset(struct aie_partition *apart)
470{
471 struct aie_device *adev = apart->adev;
472 u32 t;
473
474 for (t = AIE_TILE_TYPE_TILE; t < AIE_TILE_TYPE_MAX; t++) {
475 struct aie_tile_rscs *trscs = &apart->trscs[t];
476 struct aie_tile_attr *tattr;
477 u32 r;
478
479
480 if (t == AIE_TILE_TYPE_SHIMNOC)
481 continue;
482
483 tattr = aie_dev_get_tile_attr(adev, t);
484 for (r = AIE_RSCTYPE_PERF; r < AIE_RSCTYPE_MAX; r++) {
485 struct aie_mod_rscs *mod_rscs;
486 u32 m;
487
488 mod_rscs = trscs->mod_rscs[r];
489 if (!mod_rscs)
490 continue;
491
492 for (m = 0 ; m < tattr->num_mods; m++) {
493 struct aie_rsc_stat *rscs_stat;
494
495 rscs_stat = mod_rscs[m].rscs_stat;
496 if (!rscs_stat)
497 continue;
498
499 aie_resource_clear_all(&rscs_stat->rbits);
500 aie_resource_clear_all(&rscs_stat->sbits);
501 }
502 }
503 }
504
505
506 (void)aie_part_set_intr_rscs(apart);
507}
508
509
510
511
512
513
514
515
516
517
518
519
520
521long aie_part_rscmgr_rsc_req(struct aie_partition *apart,
522 void __user *user_args)
523{
524 struct aie_rsc_req_rsp args;
525 struct aie_location loc;
526 struct aie_rsc_stat *rstat;
527 long ret;
528 int mod_num_rscs, start_bit;
529 struct aie_rsc *rscs;
530
531 if (copy_from_user(&args, user_args, sizeof(args)))
532 return -EFAULT;
533
534 if (!args.rscs) {
535 dev_err(&apart->dev,
536 "invalid resource request, empty resources list.\n");
537 return -EINVAL;
538 }
539
540 ret = aie_part_adjust_loc(apart, args.req.loc, &loc);
541 if (ret < 0)
542 return ret;
543
544 if (args.req.type > AIE_RSCTYPE_MAX) {
545 dev_err(&apart->dev,
546 "invalid resource request, invalid resource type %d.\n",
547 args.req.type);
548 return -EINVAL;
549 }
550
551 rstat = aie_part_get_rsc_bitmaps(apart, loc, args.req.mod,
552 args.req.type);
553 start_bit = aie_part_get_rsc_startbit(apart, loc, args.req.mod,
554 args.req.type);
555 if (!rstat || start_bit < 0) {
556 dev_err(&apart->dev,
557 "invalid resource request(%u,%u), mod:%u, rsc:%u.\n",
558 args.req.loc.col, args.req.loc.row, args.req.mod,
559 args.req.type);
560 return -EINVAL;
561 }
562
563 mod_num_rscs = aie_part_get_mod_num_rscs(apart, loc, args.req.mod,
564 args.req.type);
565 if (!args.req.num_rscs || args.req.num_rscs > mod_num_rscs) {
566 dev_err(&apart->dev,
567 "invalid resource req(%u,%u),mod:%u,rsc:%u,expect=%u,max=%u.\n",
568 args.req.loc.col, args.req.loc.row, args.req.mod,
569 args.req.type, args.req.num_rscs, mod_num_rscs);
570 return -EINVAL;
571 }
572
573 rscs = kmalloc_array(args.req.num_rscs, sizeof(*rscs), GFP_KERNEL);
574 if (!rscs)
575 return -ENOMEM;
576
577 ret = mutex_lock_interruptible(&apart->mlock);
578 if (ret) {
579 kfree(rscs);
580 return ret;
581 }
582
583
584
585
586
587 if (!(args.req.flag & XAIE_RSC_PATTERN_BLOCK)) {
588 ret = aie_resource_get_common_avail(&rstat->rbits, &rstat->sbits,
589 start_bit,
590 args.req.num_rscs,
591 mod_num_rscs, rscs);
592 } else {
593 ret = aie_resource_get_common_pattern_region(&rstat->rbits,
594 &rstat->sbits,
595 start_bit,
596 args.req.num_rscs,
597 mod_num_rscs,
598 rscs);
599 }
600 mutex_unlock(&apart->mlock);
601
602 if (ret < 0) {
603 if (!(args.req.flag & XAIE_RSC_PATTERN_BLOCK)) {
604 dev_warn(&apart->dev,
605 "invalid resource req(%u,%u),mod:%u,rsc:%u,expect=%u not avail.\n",
606 args.req.loc.col, args.req.loc.row,
607 args.req.mod, args.req.type,
608 args.req.num_rscs);
609 } else {
610 dev_warn(&apart->dev,
611 "invalid contiguous resource req(%u,%u),mod:%u,rsc:%u,expect=%u not avail.\n",
612 args.req.loc.col, args.req.loc.row,
613 args.req.mod, args.req.type, args.req.num_rscs);
614 }
615 kfree(rscs);
616 return ret;
617 }
618
619 if (copy_to_user((void __user *)args.rscs, rscs,
620 sizeof(*rscs) * args.req.num_rscs))
621 ret = -EFAULT;
622 else
623 ret = 0;
624
625 kfree(rscs);
626 return ret;
627}
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642static long aie_part_rscmgr_rsc_clearbit(struct aie_partition *apart,
643 void __user *user_args,
644 bool is_release)
645{
646 struct aie_rsc args;
647 struct aie_location loc, rloc;
648 struct aie_rsc_stat *rstat;
649 long ret;
650 int mod_num_rscs, start_bit;
651
652 if (copy_from_user(&args, user_args, sizeof(args)))
653 return -EFAULT;
654
655 rloc.col = (u32)(args.loc.col & 0xFF);
656 rloc.row = (u32)(args.loc.row & 0xFF);
657 ret = aie_part_adjust_loc(apart, rloc, &loc);
658 if (ret < 0)
659 return ret;
660
661 if (args.type > AIE_RSCTYPE_MAX) {
662 dev_err(&apart->dev,
663 "invalid resource to release, invalid resource type %d.\n",
664 args.type);
665 return -EINVAL;
666 }
667
668 rstat = aie_part_get_rsc_bitmaps(apart, loc, args.mod, args.type);
669 start_bit = aie_part_get_rsc_startbit(apart, loc, args.mod,
670 args.type);
671 if (!rstat || start_bit < 0) {
672 dev_err(&apart->dev,
673 "invalid resource to release(%u,%u),mod:%u,rsc:%u.\n",
674 rloc.col, rloc.row, args.mod, args.type);
675 return -EINVAL;
676 }
677
678 mod_num_rscs = aie_part_get_mod_num_rscs(apart, loc, args.mod,
679 args.type);
680 if (args.id > mod_num_rscs) {
681 dev_err(&apart->dev,
682 "invalid resource to release(%u,%u),mod:%u,rsc:%u,id=%u.\n",
683 rloc.col, rloc.row, args.mod, args.type, args.id);
684 return -EINVAL;
685 }
686
687 ret = mutex_lock_interruptible(&apart->mlock);
688 if (ret)
689 return ret;
690
691 if (!aie_resource_testbit(&rstat->rbits, start_bit + args.id)) {
692 dev_err(&apart->dev,
693 "invalid resource to release(%u,%u),mod:%u,rsc:%u,id=%u. not requested\n",
694 rloc.col, rloc.row, args.mod, args.type, args.id);
695 mutex_unlock(&apart->mlock);
696 return -EINVAL;
697 }
698
699 aie_resource_clear(&rstat->rbits, start_bit + args.id, 1);
700 if (is_release)
701 aie_resource_clear(&rstat->sbits, start_bit + args.id, 1);
702
703 mutex_unlock(&apart->mlock);
704
705 return 0;
706}
707
708
709
710
711
712
713
714
715
716
717
718
719
720long aie_part_rscmgr_rsc_release(struct aie_partition *apart,
721 void __user *user_args)
722{
723 return aie_part_rscmgr_rsc_clearbit(apart, user_args, true);
724}
725
726
727
728
729
730
731
732
733
734
735
736
737long aie_part_rscmgr_rsc_free(struct aie_partition *apart,
738 void __user *user_args)
739{
740 return aie_part_rscmgr_rsc_clearbit(apart, user_args, false);
741}
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756long aie_part_rscmgr_rsc_req_specific(struct aie_partition *apart,
757 void __user *user_args)
758{
759 struct aie_rsc args;
760 struct aie_location loc, rloc;
761 struct aie_rsc_stat *rstat;
762 long ret;
763 int mod_num_rscs, start_bit;
764
765 if (copy_from_user(&args, user_args, sizeof(args)))
766 return -EFAULT;
767
768 rloc.col = (u32)(args.loc.col & 0xFF);
769 rloc.row = (u32)(args.loc.row & 0xFF);
770 ret = aie_part_adjust_loc(apart, rloc, &loc);
771 if (ret < 0)
772 return ret;
773
774 if (args.type > AIE_RSCTYPE_MAX) {
775 dev_err(&apart->dev,
776 "invalid resource to request, invalid resource type %d.\n",
777 args.type);
778 return -EINVAL;
779 }
780
781 rstat = aie_part_get_rsc_bitmaps(apart, loc, args.mod, args.type);
782 start_bit = aie_part_get_rsc_startbit(apart, loc, args.mod,
783 args.type);
784 if (!rstat || start_bit < 0) {
785 dev_err(&apart->dev,
786 "invalid resource to request(%u,%u),mod:%u,rsc:%u.\n",
787 rloc.col, rloc.row, args.mod, args.type);
788 return -EINVAL;
789 }
790
791 mod_num_rscs = aie_part_get_mod_num_rscs(apart, loc, args.mod,
792 args.type);
793 if (args.id > mod_num_rscs) {
794 dev_err(&apart->dev,
795 "invalid resource to request(%u,%u),mod:%u, rsc:%u,id=%u.\n",
796 rloc.col, rloc.row, args.mod, args.type, args.id);
797 return -EINVAL;
798 }
799
800 ret = mutex_lock_interruptible(&apart->mlock);
801 if (ret)
802 return ret;
803
804
805 if (aie_resource_testbit(&rstat->rbits, start_bit + args.id)) {
806 dev_err(&apart->dev,
807 "invalid resource to request(%u,%u),mod:%u,rsc:%u,id=%u, resource in use.\n",
808 rloc.col, rloc.row, args.mod, args.type, args.id);
809 mutex_unlock(&apart->mlock);
810 return -EBUSY;
811 }
812
813 aie_resource_set(&rstat->rbits, start_bit + args.id, 1);
814
815 mutex_unlock(&apart->mlock);
816
817 return 0;
818}
819
820
821
822
823
824
825
826
827
828
829
830
831
832long aie_part_rscmgr_rsc_check_avail(struct aie_partition *apart,
833 void __user *user_args)
834{
835 struct aie_rsc_stat *rstat;
836 struct aie_location loc;
837 long ret;
838 int mod_num_rscs, start_bit;
839 struct aie_rsc_req args;
840
841 if (copy_from_user(&args, user_args, sizeof(args)))
842 return -EFAULT;
843
844 ret = aie_part_adjust_loc(apart, args.loc, &loc);
845 if (ret < 0)
846 return ret;
847
848 if (args.type > AIE_RSCTYPE_MAX) {
849 dev_err(&apart->dev,
850 "invalid resource to request, invalid resource type %d.\n",
851 args.type);
852 return -EINVAL;
853 }
854
855 rstat = aie_part_get_rsc_bitmaps(apart, loc, args.mod, args.type);
856 start_bit = aie_part_get_rsc_startbit(apart, loc, args.mod,
857 args.type);
858 if (!rstat || start_bit < 0) {
859 dev_err(&apart->dev,
860 "invalid resource to request(%u,%u),mod:%u,rsc:%u.\n",
861 args.loc.col, args.loc.row, args.mod, args.type);
862 return -EINVAL;
863 }
864
865 mod_num_rscs = aie_part_get_mod_num_rscs(apart, loc, args.mod,
866 args.type);
867 ret = mutex_lock_interruptible(&apart->mlock);
868 if (ret)
869 return ret;
870
871 args.num_rscs = aie_resource_check_common_avail(&rstat->rbits,
872 &rstat->sbits,
873 start_bit,
874 mod_num_rscs);
875 mutex_unlock(&apart->mlock);
876
877 if (copy_to_user(user_args, &args, sizeof(args)))
878 return -EFAULT;
879
880 return 0;
881}
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896static int aie_part_rscmgr_get_ungated_bc_mods(struct aie_partition *apart,
897 u32 num_rscs, u32 *onum_rscs,
898 struct aie_rsc *rscs)
899{
900 struct aie_device *adev = apart->adev;
901 u32 c, r, i = 0;
902
903 for (c = 0; c < apart->range.size.col; c++) {
904 for (r = 0; r < apart->range.size.row; r++) {
905 struct aie_location l;
906 u32 ttype, m;
907 const struct aie_tile_attr *tattr;
908 const struct aie_tile_rsc_attr *rattr;
909 enum aie_rsc_type rtype = AIE_RSCTYPE_BROADCAST;
910
911 l.col = apart->range.start.col + c;
912 l.row = r;
913 ttype = adev->ops->get_tile_type(&l);
914 tattr = &adev->ttype_attr[ttype];
915 rattr = &tattr->rscs_attr[rtype];
916 for (m = 0; m < tattr->num_mods; m++) {
917
918
919
920
921 if (!rattr->mod_attr[m].num_rscs)
922 continue;
923
924 if (aie_part_check_clk_enable_loc(apart, &l)) {
925 if (i >= num_rscs) {
926 dev_err(&apart->dev,
927 "failed to returns all ungated tiles, not enough resource elements.\n");
928 return -EINVAL;
929 }
930 rscs[i].loc.col = (u8)(c & 0xFF);
931 rscs[i].loc.row = (u8)(r & 0xFF);
932 rscs[i].mod = tattr->mods[m];
933 i++;
934 }
935 }
936 }
937 }
938
939 *onum_rscs = i;
940 return 0;
941}
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959static int aie_part_rscmgr_get_or_bc_stat(struct aie_partition *apart,
960 u32 num_rscs, struct aie_rsc *rscs,
961 bool runtime_only,
962 unsigned long *or_stat)
963{
964 u32 i;
965
966 *or_stat = 0;
967 for (i = 0; i < num_rscs; i++) {
968 struct aie_location l;
969 struct aie_rsc_stat *rstat;
970 int mod_num_rscs, start_bit;
971
972 l.col = apart->range.start.col + rscs[i].loc.col;
973 l.row = rscs[i].loc.row;
974 rstat = aie_part_get_rsc_bitmaps(apart, l, rscs[i].mod,
975 AIE_RSCTYPE_BROADCAST);
976 start_bit = aie_part_get_rsc_startbit(apart, l, rscs[i].mod,
977 AIE_RSCTYPE_BROADCAST);
978 if (!rstat || start_bit < 0) {
979 dev_err(&apart->dev,
980 "failed to get broadcast bitmap for[%u]:tile(%u,%u), mod=%u.\n",
981 i, rscs[i].loc.col, rscs[i].loc.row,
982 rscs[i].mod);
983 return -EINVAL;
984 }
985 mod_num_rscs = aie_part_get_mod_num_rscs(apart, l, rscs[i].mod,
986 AIE_RSCTYPE_BROADCAST);
987 *or_stat |= aie_resource_or_get_valueul(&rstat->rbits,
988 start_bit,
989 mod_num_rscs);
990 if (!runtime_only)
991 *or_stat |= aie_resource_or_get_valueul(&rstat->sbits,
992 start_bit,
993 mod_num_rscs);
994 }
995
996 return 0;
997}
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012static int aie_part_rscmgr_get_common_bc(struct aie_partition *apart,
1013 u32 num_rscs, struct aie_rsc *rscs)
1014{
1015 unsigned long or_stat, b;
1016 int ret;
1017 struct aie_location l;
1018 int mod_num_rscs;
1019
1020 l.col = apart->range.start.col + (u32)rscs[0].loc.row;
1021 l.row = (u32)rscs[0].loc.row;
1022
1023 ret = aie_part_rscmgr_get_or_bc_stat(apart, num_rscs, rscs, false,
1024 &or_stat);
1025 if (ret)
1026 return ret;
1027
1028 mod_num_rscs = aie_part_get_mod_num_rscs(apart, l, rscs[0].mod,
1029 AIE_RSCTYPE_BROADCAST);
1030 b = bitmap_find_next_zero_area(&or_stat, mod_num_rscs, 0, 1, 0);
1031 if (b >= mod_num_rscs)
1032 return -EINVAL;
1033
1034 return (int)b;
1035}
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053static int aie_part_rscmgr_check_common_bc(struct aie_partition *apart,
1054 u32 bc, u32 num_rscs,
1055 struct aie_rsc *rscs)
1056{
1057 unsigned long or_stat;
1058 int ret;
1059 struct aie_location l;
1060 int mod_num_rscs;
1061
1062 l.col = apart->range.start.col + (u32)rscs[0].loc.row;
1063 l.row = (u32)rscs[0].loc.row;
1064
1065 mod_num_rscs = aie_part_get_mod_num_rscs(apart, l, rscs[0].mod,
1066 AIE_RSCTYPE_BROADCAST);
1067 if (bc > mod_num_rscs) {
1068 dev_err(&apart->dev,
1069 "invalid specified broadcast id %u, max is %u.\n",
1070 bc, mod_num_rscs);
1071 return -EINVAL;
1072 }
1073
1074 ret = aie_part_rscmgr_get_or_bc_stat(apart, num_rscs, rscs, true,
1075 &or_stat);
1076 if (ret)
1077 return ret;
1078
1079 if (test_bit(bc, &or_stat)) {
1080 dev_err(&apart->dev,
1081 "specified broadcast id %u is occupied.\n", bc);
1082 return -EBUSY;
1083 }
1084
1085 return 0;
1086}
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102static int aie_part_rscmgr_check_rscs_modules(struct aie_partition *apart,
1103 u32 num_rscs,
1104 struct aie_rsc *rscs)
1105{
1106 struct aie_device *adev = apart->adev;
1107 u32 i;
1108
1109 for (i = 0; i < num_rscs; i++) {
1110 struct aie_location l;
1111
1112 l.col = apart->range.start.col + rscs[i].loc.col;
1113 l.row = rscs[i].loc.row;
1114
1115 if (aie_validate_location(apart, l)) {
1116 dev_err(&apart->dev,
1117 "failed resource check tile(%u,%u) invalid.\n",
1118 rscs[i].loc.col, rscs[i].loc.row);
1119 return -EINVAL;
1120 }
1121
1122
1123 if (aie_dev_get_mod_id(adev, adev->ops->get_tile_type(&l),
1124 rscs[i].mod) < 0) {
1125 dev_err(&apart->dev,
1126 "failed resource check, tile(%u,%u) mod %u invalid.\n",
1127 rscs[i].loc.col, rscs[i].loc.row,
1128 rscs[i].mod);
1129 return -EINVAL;
1130 }
1131
1132
1133 if (!aie_part_check_clk_enable_loc(apart, &l)) {
1134 dev_err(&apart->dev,
1135 "failed resource check, tile(%u,%u) mod=%u is gated.\n",
1136 rscs[i].loc.col, rscs[i].loc.row,
1137 rscs[i].mod);
1138 return -EINVAL;
1139 }
1140 }
1141
1142 return 0;
1143}
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158int aie_part_rscmgr_set_tile_broadcast(struct aie_partition *apart,
1159 struct aie_location loc,
1160 enum aie_module_type mod, uint32_t id)
1161{
1162 struct aie_rsc_stat *rstat;
1163 int start_bit;
1164
1165 rstat = aie_part_get_rsc_bitmaps(apart, loc, mod,
1166 AIE_RSCTYPE_BROADCAST);
1167
1168 if (WARN_ON(!rstat || !rstat->rbits.bitmap))
1169 return -EFAULT;
1170
1171 start_bit = aie_part_get_rsc_startbit(apart, loc, mod,
1172 AIE_RSCTYPE_BROADCAST);
1173 aie_resource_set(&rstat->rbits, start_bit + id, 1);
1174
1175 return 0;
1176}
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200long aie_part_rscmgr_get_broadcast(struct aie_partition *apart,
1201 void __user *user_args)
1202{
1203 struct aie_rsc_bc_req args;
1204 struct aie_rsc *rscs;
1205 u32 i;
1206 long ret;
1207
1208 if (copy_from_user(&args, user_args, sizeof(args)))
1209 return -EFAULT;
1210
1211 rscs = kmalloc_array(args.num_rscs, sizeof(*rscs), GFP_KERNEL);
1212 if (!rscs)
1213 return -ENOMEM;
1214
1215 if (!(args.flag & XAIE_BROADCAST_ALL)) {
1216 if (copy_from_user(rscs, (void __user *)args.rscs,
1217 sizeof(*rscs) * args.num_rscs)) {
1218 kfree(rscs);
1219 return -EFAULT;
1220 }
1221 }
1222
1223 ret = mutex_lock_interruptible(&apart->mlock);
1224 if (ret) {
1225 kfree(rscs);
1226 return ret;
1227 }
1228
1229 if (args.flag & XAIE_BROADCAST_ALL)
1230
1231
1232
1233
1234 ret = aie_part_rscmgr_get_ungated_bc_mods(apart, args.num_rscs,
1235 &args.num_rscs,
1236 rscs);
1237 else
1238
1239
1240
1241
1242 ret = aie_part_rscmgr_check_rscs_modules(apart, args.num_rscs,
1243 rscs);
1244 if (ret)
1245 goto error;
1246
1247
1248 if (args.id == XAIE_BROADCAST_ID_ANY) {
1249 ret = aie_part_rscmgr_get_common_bc(apart, args.num_rscs, rscs);
1250 if (ret >= 0) {
1251 args.id = (u32)ret;
1252 ret = 0;
1253 } else {
1254 dev_warn(&apart->dev, "no available broadcast channel.\n");
1255 }
1256 } else {
1257 ret = aie_part_rscmgr_check_common_bc(apart, args.id,
1258 args.num_rscs,
1259 rscs);
1260 }
1261 if (ret)
1262 goto error;
1263
1264
1265 for (i = 0; i < args.num_rscs; i++) {
1266 struct aie_location l;
1267
1268 l.col = apart->range.start.col + rscs[i].loc.col;
1269 l.row = rscs[i].loc.row;
1270 ret = aie_part_rscmgr_set_tile_broadcast(apart, l, rscs[i].mod,
1271 args.id);
1272 if (ret)
1273 goto error;
1274
1275 rscs[i].id = args.id;
1276 }
1277
1278 mutex_unlock(&apart->mlock);
1279
1280 if (copy_to_user((void __user *)args.rscs, rscs,
1281 sizeof(*rscs) * args.num_rscs)) {
1282 kfree(rscs);
1283 return -EFAULT;
1284 }
1285
1286
1287
1288
1289
1290
1291 if (args.flag & XAIE_BROADCAST_ALL) {
1292 struct aie_rsc_bc_req __user *uargs = user_args;
1293
1294 if (copy_to_user((void __user *)&uargs->num_rscs,
1295 &args.num_rscs, sizeof(args.num_rscs))) {
1296 kfree(rscs);
1297 return -EFAULT;
1298 }
1299 }
1300
1301 kfree(rscs);
1302 return 0;
1303error:
1304 mutex_unlock(&apart->mlock);
1305 kfree(rscs);
1306 return ret;
1307}
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320int aie_part_rscmgr_set_static(struct aie_partition *apart, void *meta)
1321{
1322 struct aie_rsc_meta_header *header = meta;
1323 struct aie_rsc_bitmap *bitmap;
1324 u64 i, num_bitmaps, offset;
1325
1326 if (!header) {
1327 dev_err(&apart->dev,
1328 "failed to get static resources, meta data is NULL.\n");
1329 return -EINVAL;
1330 }
1331
1332
1333
1334
1335
1336 num_bitmaps = header->stat;
1337 offset = header->bitmap_off;
1338 if (!num_bitmaps || offset < sizeof(*header)) {
1339 dev_err(&apart->dev,
1340 "failed to get static resources, invalid header.\n");
1341 return -EINVAL;
1342 }
1343
1344 bitmap = (struct aie_rsc_bitmap *)(meta + offset);
1345 for (i = 0; i < num_bitmaps; i++) {
1346 struct aie_rsc_stat *rstat;
1347 const struct aie_mod_rsc_attr *mrattr;
1348 u64 header = bitmap->header;
1349 u32 lrlen, rlen, ttype, mtype, rtype, total;
1350
1351 ttype = AIE_RSC_BITMAP_HEAD_VAL(TILETYPE, header);
1352 mtype = AIE_RSC_BITMAP_HEAD_VAL(MODTYPE, header);
1353 rtype = AIE_RSC_BITMAP_HEAD_VAL(RSCTYPE, header);
1354 rlen = AIE_RSC_BITMAP_HEAD_VAL(LENU64, header);
1355
1356 if (!rlen) {
1357 dev_err(&apart->dev,
1358 "invalid static bitmap[%llu], length is 0.\n",
1359 i);
1360 return -EINVAL;
1361 }
1362
1363 mrattr = aie_dev_get_mod_rsc_attr(apart->adev, ttype, mtype,
1364 rtype);
1365 if (!mrattr) {
1366 dev_err(&apart->dev,
1367 "invalid static bitmap[%llu], invalid tile(%u)/module(%u)/rsce(%u) types combination.\n",
1368 i, ttype, mtype, rtype);
1369 return -EINVAL;
1370 }
1371
1372 total = mrattr->num_rscs * apart->range.size.col *
1373 aie_part_get_tile_rows(apart, ttype);
1374 lrlen = BITS_TO_LONGS(total);
1375 if (rlen != lrlen) {
1376 dev_err(&apart->dev,
1377 "invalid static bitmap[%llu], tile(%u)/module(%u)/rscs(%u), expect len(%u), actual(%u).\n",
1378 i, ttype, mtype, rtype, lrlen, rlen);
1379 return -EINVAL;
1380 }
1381
1382 rstat = aie_part_get_ttype_rsc_bitmaps(apart, ttype, mtype,
1383 rtype);
1384
1385 if (WARN_ON(!rstat || !rstat->sbits.bitmap))
1386 return -EFAULT;
1387
1388
1389 bitmap_copy(rstat->sbits.bitmap,
1390 (unsigned long *)bitmap->bitmap, total);
1391
1392 bitmap = (struct aie_rsc_bitmap *)((void *)bitmap +
1393 sizeof(header) +
1394 rlen * sizeof(u64));
1395 }
1396
1397 return 0;
1398}
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413static int aie_part_rscmgr_check_static(struct aie_rsc_stat *rstat,
1414 u32 sbit, u32 total)
1415{
1416 u32 i;
1417 int num_static = 0;
1418
1419 for (i = sbit; i < sbit + total; i++) {
1420 if (aie_resource_testbit(&rstat->sbits, i))
1421 num_static++;
1422 }
1423
1424 return num_static;
1425}
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441static int aie_part_rscmgr_check_avail(struct aie_rsc_stat *rstat,
1442 u32 sbit, u32 total)
1443{
1444 return aie_resource_check_common_avail(&rstat->rbits,
1445 &rstat->sbits,
1446 sbit, total);
1447}
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469long aie_part_rscmgr_get_statistics(struct aie_partition *apart,
1470 void __user *user_args)
1471{
1472 struct aie_rsc_user_stat_array args;
1473 struct aie_rsc_user_stat __user *ustat_ptr;
1474 u32 i;
1475
1476 if (copy_from_user(&args, user_args, sizeof(args)))
1477 return -EFAULT;
1478
1479 if (args.stats_type >= AIE_RSC_STAT_TYPE_MAX) {
1480 dev_err(&apart->dev,
1481 "get rsc statistics failed, invalid rsc stat type %u.\n",
1482 args.stats_type);
1483 return -EINVAL;
1484 }
1485
1486 ustat_ptr = (struct aie_rsc_user_stat __user *)args.stats;
1487 for (i = 0; i < args.num_stats; i++) {
1488 struct aie_rsc_user_stat ustat;
1489 struct aie_rsc_stat *rstat;
1490 struct aie_location rloc, loc;
1491 long ret;
1492 int max_rscs, start_bit;
1493
1494 if (copy_from_user(&ustat, (void __user *)ustat_ptr,
1495 sizeof(ustat)))
1496 return -EFAULT;
1497
1498
1499 rloc.col = (u32)(ustat.loc.col & 0xFF);
1500 rloc.row = (u32)(ustat.loc.row & 0xFF);
1501 ret = aie_part_adjust_loc(apart, rloc, &loc);
1502 if (ret < 0)
1503 return ret;
1504
1505 if (ustat.type > AIE_RSCTYPE_MAX) {
1506 dev_err(&apart->dev,
1507 "get rsc statistics failed, invalid resource type %d.\n",
1508 ustat.type);
1509 return -EINVAL;
1510 }
1511
1512 rstat = aie_part_get_rsc_bitmaps(apart, loc, ustat.mod,
1513 ustat.type);
1514 start_bit = aie_part_get_rsc_startbit(apart, loc, ustat.mod,
1515 ustat.type);
1516 if (!rstat || start_bit < 0) {
1517 dev_err(&apart->dev,
1518 "get rsc statistics failed, invalid resource(%u,%u),mod:%u,rsc:%u.\n",
1519 loc.col, loc.row, ustat.mod, ustat.type);
1520 return -EINVAL;
1521 }
1522
1523 max_rscs = aie_part_get_mod_num_rscs(apart, loc, ustat.mod,
1524 ustat.type);
1525 ret = mutex_lock_interruptible(&apart->mlock);
1526 if (ret)
1527 return ret;
1528
1529 if (args.stats_type == AIE_RSC_STAT_TYPE_STATIC)
1530 ustat.num_rscs = aie_part_rscmgr_check_static(rstat,
1531 start_bit,
1532 max_rscs);
1533 else
1534 ustat.num_rscs = aie_part_rscmgr_check_avail(rstat,
1535 start_bit,
1536 max_rscs);
1537
1538 mutex_unlock(&apart->mlock);
1539 if (WARN_ON(ustat.num_rscs < 0))
1540 return -EFAULT;
1541
1542
1543 if (copy_to_user((void __user *)ustat_ptr, &ustat,
1544 sizeof(ustat)))
1545 return -EFAULT;
1546
1547 ustat_ptr++;
1548 }
1549
1550 return 0;
1551}
1552