linux/drivers/infiniband/hw/hfi1/affinity.c
<<
>>
Prefs
   1/*
   2 * Copyright(c) 2015 - 2018 Intel Corporation.
   3 *
   4 * This file is provided under a dual BSD/GPLv2 license.  When using or
   5 * redistributing this file, you may do so under either license.
   6 *
   7 * GPL LICENSE SUMMARY
   8 *
   9 * This program is free software; you can redistribute it and/or modify
  10 * it under the terms of version 2 of the GNU General Public License as
  11 * published by the Free Software Foundation.
  12 *
  13 * This program is distributed in the hope that it will be useful, but
  14 * WITHOUT ANY WARRANTY; without even the implied warranty of
  15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  16 * General Public License for more details.
  17 *
  18 * BSD LICENSE
  19 *
  20 * Redistribution and use in source and binary forms, with or without
  21 * modification, are permitted provided that the following conditions
  22 * are met:
  23 *
  24 *  - Redistributions of source code must retain the above copyright
  25 *    notice, this list of conditions and the following disclaimer.
  26 *  - Redistributions in binary form must reproduce the above copyright
  27 *    notice, this list of conditions and the following disclaimer in
  28 *    the documentation and/or other materials provided with the
  29 *    distribution.
  30 *  - Neither the name of Intel Corporation nor the names of its
  31 *    contributors may be used to endorse or promote products derived
  32 *    from this software without specific prior written permission.
  33 *
  34 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  35 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  36 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  37 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  38 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  39 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  40 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  41 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  42 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  43 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  44 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  45 *
  46 */
  47#include <linux/topology.h>
  48#include <linux/cpumask.h>
  49#include <linux/module.h>
  50#include <linux/interrupt.h>
  51#include <linux/numa.h>
  52
  53#include "hfi.h"
  54#include "affinity.h"
  55#include "sdma.h"
  56#include "trace.h"
  57
  58struct hfi1_affinity_node_list node_affinity = {
  59        .list = LIST_HEAD_INIT(node_affinity.list),
  60        .lock = __MUTEX_INITIALIZER(node_affinity.lock)
  61};
  62
  63/* Name of IRQ types, indexed by enum irq_type */
  64static const char * const irq_type_names[] = {
  65        "SDMA",
  66        "RCVCTXT",
  67        "GENERAL",
  68        "OTHER",
  69};
  70
  71/* Per NUMA node count of HFI devices */
  72static unsigned int *hfi1_per_node_cntr;
  73
  74static inline void init_cpu_mask_set(struct cpu_mask_set *set)
  75{
  76        cpumask_clear(&set->mask);
  77        cpumask_clear(&set->used);
  78        set->gen = 0;
  79}
  80
  81/* Increment generation of CPU set if needed */
  82static void _cpu_mask_set_gen_inc(struct cpu_mask_set *set)
  83{
  84        if (cpumask_equal(&set->mask, &set->used)) {
  85                /*
  86                 * We've used up all the CPUs, bump up the generation
  87                 * and reset the 'used' map
  88                 */
  89                set->gen++;
  90                cpumask_clear(&set->used);
  91        }
  92}
  93
  94static void _cpu_mask_set_gen_dec(struct cpu_mask_set *set)
  95{
  96        if (cpumask_empty(&set->used) && set->gen) {
  97                set->gen--;
  98                cpumask_copy(&set->used, &set->mask);
  99        }
 100}
 101
 102/* Get the first CPU from the list of unused CPUs in a CPU set data structure */
 103static int cpu_mask_set_get_first(struct cpu_mask_set *set, cpumask_var_t diff)
 104{
 105        int cpu;
 106
 107        if (!diff || !set)
 108                return -EINVAL;
 109
 110        _cpu_mask_set_gen_inc(set);
 111
 112        /* Find out CPUs left in CPU mask */
 113        cpumask_andnot(diff, &set->mask, &set->used);
 114
 115        cpu = cpumask_first(diff);
 116        if (cpu >= nr_cpu_ids) /* empty */
 117                cpu = -EINVAL;
 118        else
 119                cpumask_set_cpu(cpu, &set->used);
 120
 121        return cpu;
 122}
 123
 124static void cpu_mask_set_put(struct cpu_mask_set *set, int cpu)
 125{
 126        if (!set)
 127                return;
 128
 129        cpumask_clear_cpu(cpu, &set->used);
 130        _cpu_mask_set_gen_dec(set);
 131}
 132
 133/* Initialize non-HT cpu cores mask */
 134void init_real_cpu_mask(void)
 135{
 136        int possible, curr_cpu, i, ht;
 137
 138        cpumask_clear(&node_affinity.real_cpu_mask);
 139
 140        /* Start with cpu online mask as the real cpu mask */
 141        cpumask_copy(&node_affinity.real_cpu_mask, cpu_online_mask);
 142
 143        /*
 144         * Remove HT cores from the real cpu mask.  Do this in two steps below.
 145         */
 146        possible = cpumask_weight(&node_affinity.real_cpu_mask);
 147        ht = cpumask_weight(topology_sibling_cpumask(
 148                                cpumask_first(&node_affinity.real_cpu_mask)));
 149        /*
 150         * Step 1.  Skip over the first N HT siblings and use them as the
 151         * "real" cores.  Assumes that HT cores are not enumerated in
 152         * succession (except in the single core case).
 153         */
 154        curr_cpu = cpumask_first(&node_affinity.real_cpu_mask);
 155        for (i = 0; i < possible / ht; i++)
 156                curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
 157        /*
 158         * Step 2.  Remove the remaining HT siblings.  Use cpumask_next() to
 159         * skip any gaps.
 160         */
 161        for (; i < possible; i++) {
 162                cpumask_clear_cpu(curr_cpu, &node_affinity.real_cpu_mask);
 163                curr_cpu = cpumask_next(curr_cpu, &node_affinity.real_cpu_mask);
 164        }
 165}
 166
 167int node_affinity_init(void)
 168{
 169        int node;
 170        struct pci_dev *dev = NULL;
 171        const struct pci_device_id *ids = hfi1_pci_tbl;
 172
 173        cpumask_clear(&node_affinity.proc.used);
 174        cpumask_copy(&node_affinity.proc.mask, cpu_online_mask);
 175
 176        node_affinity.proc.gen = 0;
 177        node_affinity.num_core_siblings =
 178                                cpumask_weight(topology_sibling_cpumask(
 179                                        cpumask_first(&node_affinity.proc.mask)
 180                                        ));
 181        node_affinity.num_possible_nodes = num_possible_nodes();
 182        node_affinity.num_online_nodes = num_online_nodes();
 183        node_affinity.num_online_cpus = num_online_cpus();
 184
 185        /*
 186         * The real cpu mask is part of the affinity struct but it has to be
 187         * initialized early. It is needed to calculate the number of user
 188         * contexts in set_up_context_variables().
 189         */
 190        init_real_cpu_mask();
 191
 192        hfi1_per_node_cntr = kcalloc(node_affinity.num_possible_nodes,
 193                                     sizeof(*hfi1_per_node_cntr), GFP_KERNEL);
 194        if (!hfi1_per_node_cntr)
 195                return -ENOMEM;
 196
 197        while (ids->vendor) {
 198                dev = NULL;
 199                while ((dev = pci_get_device(ids->vendor, ids->device, dev))) {
 200                        node = pcibus_to_node(dev->bus);
 201                        if (node < 0)
 202                                goto out;
 203
 204                        hfi1_per_node_cntr[node]++;
 205                }
 206                ids++;
 207        }
 208
 209        return 0;
 210
 211out:
 212        /*
 213         * Invalid PCI NUMA node information found, note it, and populate
 214         * our database 1:1.
 215         */
 216        pr_err("HFI: Invalid PCI NUMA node. Performance may be affected\n");
 217        pr_err("HFI: System BIOS may need to be upgraded\n");
 218        for (node = 0; node < node_affinity.num_possible_nodes; node++)
 219                hfi1_per_node_cntr[node] = 1;
 220
 221        return 0;
 222}
 223
 224static void node_affinity_destroy(struct hfi1_affinity_node *entry)
 225{
 226        free_percpu(entry->comp_vect_affinity);
 227        kfree(entry);
 228}
 229
 230void node_affinity_destroy_all(void)
 231{
 232        struct list_head *pos, *q;
 233        struct hfi1_affinity_node *entry;
 234
 235        mutex_lock(&node_affinity.lock);
 236        list_for_each_safe(pos, q, &node_affinity.list) {
 237                entry = list_entry(pos, struct hfi1_affinity_node,
 238                                   list);
 239                list_del(pos);
 240                node_affinity_destroy(entry);
 241        }
 242        mutex_unlock(&node_affinity.lock);
 243        kfree(hfi1_per_node_cntr);
 244}
 245
 246static struct hfi1_affinity_node *node_affinity_allocate(int node)
 247{
 248        struct hfi1_affinity_node *entry;
 249
 250        entry = kzalloc(sizeof(*entry), GFP_KERNEL);
 251        if (!entry)
 252                return NULL;
 253        entry->node = node;
 254        entry->comp_vect_affinity = alloc_percpu(u16);
 255        INIT_LIST_HEAD(&entry->list);
 256
 257        return entry;
 258}
 259
 260/*
 261 * It appends an entry to the list.
 262 * It *must* be called with node_affinity.lock held.
 263 */
 264static void node_affinity_add_tail(struct hfi1_affinity_node *entry)
 265{
 266        list_add_tail(&entry->list, &node_affinity.list);
 267}
 268
 269/* It must be called with node_affinity.lock held */
 270static struct hfi1_affinity_node *node_affinity_lookup(int node)
 271{
 272        struct list_head *pos;
 273        struct hfi1_affinity_node *entry;
 274
 275        list_for_each(pos, &node_affinity.list) {
 276                entry = list_entry(pos, struct hfi1_affinity_node, list);
 277                if (entry->node == node)
 278                        return entry;
 279        }
 280
 281        return NULL;
 282}
 283
 284static int per_cpu_affinity_get(cpumask_var_t possible_cpumask,
 285                                u16 __percpu *comp_vect_affinity)
 286{
 287        int curr_cpu;
 288        u16 cntr;
 289        u16 prev_cntr;
 290        int ret_cpu;
 291
 292        if (!possible_cpumask) {
 293                ret_cpu = -EINVAL;
 294                goto fail;
 295        }
 296
 297        if (!comp_vect_affinity) {
 298                ret_cpu = -EINVAL;
 299                goto fail;
 300        }
 301
 302        ret_cpu = cpumask_first(possible_cpumask);
 303        if (ret_cpu >= nr_cpu_ids) {
 304                ret_cpu = -EINVAL;
 305                goto fail;
 306        }
 307
 308        prev_cntr = *per_cpu_ptr(comp_vect_affinity, ret_cpu);
 309        for_each_cpu(curr_cpu, possible_cpumask) {
 310                cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
 311
 312                if (cntr < prev_cntr) {
 313                        ret_cpu = curr_cpu;
 314                        prev_cntr = cntr;
 315                }
 316        }
 317
 318        *per_cpu_ptr(comp_vect_affinity, ret_cpu) += 1;
 319
 320fail:
 321        return ret_cpu;
 322}
 323
 324static int per_cpu_affinity_put_max(cpumask_var_t possible_cpumask,
 325                                    u16 __percpu *comp_vect_affinity)
 326{
 327        int curr_cpu;
 328        int max_cpu;
 329        u16 cntr;
 330        u16 prev_cntr;
 331
 332        if (!possible_cpumask)
 333                return -EINVAL;
 334
 335        if (!comp_vect_affinity)
 336                return -EINVAL;
 337
 338        max_cpu = cpumask_first(possible_cpumask);
 339        if (max_cpu >= nr_cpu_ids)
 340                return -EINVAL;
 341
 342        prev_cntr = *per_cpu_ptr(comp_vect_affinity, max_cpu);
 343        for_each_cpu(curr_cpu, possible_cpumask) {
 344                cntr = *per_cpu_ptr(comp_vect_affinity, curr_cpu);
 345
 346                if (cntr > prev_cntr) {
 347                        max_cpu = curr_cpu;
 348                        prev_cntr = cntr;
 349                }
 350        }
 351
 352        *per_cpu_ptr(comp_vect_affinity, max_cpu) -= 1;
 353
 354        return max_cpu;
 355}
 356
 357/*
 358 * Non-interrupt CPUs are used first, then interrupt CPUs.
 359 * Two already allocated cpu masks must be passed.
 360 */
 361static int _dev_comp_vect_cpu_get(struct hfi1_devdata *dd,
 362                                  struct hfi1_affinity_node *entry,
 363                                  cpumask_var_t non_intr_cpus,
 364                                  cpumask_var_t available_cpus)
 365        __must_hold(&node_affinity.lock)
 366{
 367        int cpu;
 368        struct cpu_mask_set *set = dd->comp_vect;
 369
 370        lockdep_assert_held(&node_affinity.lock);
 371        if (!non_intr_cpus) {
 372                cpu = -1;
 373                goto fail;
 374        }
 375
 376        if (!available_cpus) {
 377                cpu = -1;
 378                goto fail;
 379        }
 380
 381        /* Available CPUs for pinning completion vectors */
 382        _cpu_mask_set_gen_inc(set);
 383        cpumask_andnot(available_cpus, &set->mask, &set->used);
 384
 385        /* Available CPUs without SDMA engine interrupts */
 386        cpumask_andnot(non_intr_cpus, available_cpus,
 387                       &entry->def_intr.used);
 388
 389        /* If there are non-interrupt CPUs available, use them first */
 390        if (!cpumask_empty(non_intr_cpus))
 391                cpu = cpumask_first(non_intr_cpus);
 392        else /* Otherwise, use interrupt CPUs */
 393                cpu = cpumask_first(available_cpus);
 394
 395        if (cpu >= nr_cpu_ids) { /* empty */
 396                cpu = -1;
 397                goto fail;
 398        }
 399        cpumask_set_cpu(cpu, &set->used);
 400
 401fail:
 402        return cpu;
 403}
 404
 405static void _dev_comp_vect_cpu_put(struct hfi1_devdata *dd, int cpu)
 406{
 407        struct cpu_mask_set *set = dd->comp_vect;
 408
 409        if (cpu < 0)
 410                return;
 411
 412        cpu_mask_set_put(set, cpu);
 413}
 414
 415/* _dev_comp_vect_mappings_destroy() is reentrant */
 416static void _dev_comp_vect_mappings_destroy(struct hfi1_devdata *dd)
 417{
 418        int i, cpu;
 419
 420        if (!dd->comp_vect_mappings)
 421                return;
 422
 423        for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
 424                cpu = dd->comp_vect_mappings[i];
 425                _dev_comp_vect_cpu_put(dd, cpu);
 426                dd->comp_vect_mappings[i] = -1;
 427                hfi1_cdbg(AFFINITY,
 428                          "[%s] Release CPU %d from completion vector %d",
 429                          rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), cpu, i);
 430        }
 431
 432        kfree(dd->comp_vect_mappings);
 433        dd->comp_vect_mappings = NULL;
 434}
 435
 436/*
 437 * This function creates the table for looking up CPUs for completion vectors.
 438 * num_comp_vectors needs to have been initilized before calling this function.
 439 */
 440static int _dev_comp_vect_mappings_create(struct hfi1_devdata *dd,
 441                                          struct hfi1_affinity_node *entry)
 442        __must_hold(&node_affinity.lock)
 443{
 444        int i, cpu, ret;
 445        cpumask_var_t non_intr_cpus;
 446        cpumask_var_t available_cpus;
 447
 448        lockdep_assert_held(&node_affinity.lock);
 449
 450        if (!zalloc_cpumask_var(&non_intr_cpus, GFP_KERNEL))
 451                return -ENOMEM;
 452
 453        if (!zalloc_cpumask_var(&available_cpus, GFP_KERNEL)) {
 454                free_cpumask_var(non_intr_cpus);
 455                return -ENOMEM;
 456        }
 457
 458        dd->comp_vect_mappings = kcalloc(dd->comp_vect_possible_cpus,
 459                                         sizeof(*dd->comp_vect_mappings),
 460                                         GFP_KERNEL);
 461        if (!dd->comp_vect_mappings) {
 462                ret = -ENOMEM;
 463                goto fail;
 464        }
 465        for (i = 0; i < dd->comp_vect_possible_cpus; i++)
 466                dd->comp_vect_mappings[i] = -1;
 467
 468        for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
 469                cpu = _dev_comp_vect_cpu_get(dd, entry, non_intr_cpus,
 470                                             available_cpus);
 471                if (cpu < 0) {
 472                        ret = -EINVAL;
 473                        goto fail;
 474                }
 475
 476                dd->comp_vect_mappings[i] = cpu;
 477                hfi1_cdbg(AFFINITY,
 478                          "[%s] Completion Vector %d -> CPU %d",
 479                          rvt_get_ibdev_name(&(dd)->verbs_dev.rdi), i, cpu);
 480        }
 481
 482        return 0;
 483
 484fail:
 485        free_cpumask_var(available_cpus);
 486        free_cpumask_var(non_intr_cpus);
 487        _dev_comp_vect_mappings_destroy(dd);
 488
 489        return ret;
 490}
 491
 492int hfi1_comp_vectors_set_up(struct hfi1_devdata *dd)
 493{
 494        int ret;
 495        struct hfi1_affinity_node *entry;
 496
 497        mutex_lock(&node_affinity.lock);
 498        entry = node_affinity_lookup(dd->node);
 499        if (!entry) {
 500                ret = -EINVAL;
 501                goto unlock;
 502        }
 503        ret = _dev_comp_vect_mappings_create(dd, entry);
 504unlock:
 505        mutex_unlock(&node_affinity.lock);
 506
 507        return ret;
 508}
 509
 510void hfi1_comp_vectors_clean_up(struct hfi1_devdata *dd)
 511{
 512        _dev_comp_vect_mappings_destroy(dd);
 513}
 514
 515int hfi1_comp_vect_mappings_lookup(struct rvt_dev_info *rdi, int comp_vect)
 516{
 517        struct hfi1_ibdev *verbs_dev = dev_from_rdi(rdi);
 518        struct hfi1_devdata *dd = dd_from_dev(verbs_dev);
 519
 520        if (!dd->comp_vect_mappings)
 521                return -EINVAL;
 522        if (comp_vect >= dd->comp_vect_possible_cpus)
 523                return -EINVAL;
 524
 525        return dd->comp_vect_mappings[comp_vect];
 526}
 527
 528/*
 529 * It assumes dd->comp_vect_possible_cpus is available.
 530 */
 531static int _dev_comp_vect_cpu_mask_init(struct hfi1_devdata *dd,
 532                                        struct hfi1_affinity_node *entry,
 533                                        bool first_dev_init)
 534        __must_hold(&node_affinity.lock)
 535{
 536        int i, j, curr_cpu;
 537        int possible_cpus_comp_vect = 0;
 538        struct cpumask *dev_comp_vect_mask = &dd->comp_vect->mask;
 539
 540        lockdep_assert_held(&node_affinity.lock);
 541        /*
 542         * If there's only one CPU available for completion vectors, then
 543         * there will only be one completion vector available. Othewise,
 544         * the number of completion vector available will be the number of
 545         * available CPUs divide it by the number of devices in the
 546         * local NUMA node.
 547         */
 548        if (cpumask_weight(&entry->comp_vect_mask) == 1) {
 549                possible_cpus_comp_vect = 1;
 550                dd_dev_warn(dd,
 551                            "Number of kernel receive queues is too large for completion vector affinity to be effective\n");
 552        } else {
 553                possible_cpus_comp_vect +=
 554                        cpumask_weight(&entry->comp_vect_mask) /
 555                                       hfi1_per_node_cntr[dd->node];
 556
 557                /*
 558                 * If the completion vector CPUs available doesn't divide
 559                 * evenly among devices, then the first device device to be
 560                 * initialized gets an extra CPU.
 561                 */
 562                if (first_dev_init &&
 563                    cpumask_weight(&entry->comp_vect_mask) %
 564                    hfi1_per_node_cntr[dd->node] != 0)
 565                        possible_cpus_comp_vect++;
 566        }
 567
 568        dd->comp_vect_possible_cpus = possible_cpus_comp_vect;
 569
 570        /* Reserving CPUs for device completion vector */
 571        for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
 572                curr_cpu = per_cpu_affinity_get(&entry->comp_vect_mask,
 573                                                entry->comp_vect_affinity);
 574                if (curr_cpu < 0)
 575                        goto fail;
 576
 577                cpumask_set_cpu(curr_cpu, dev_comp_vect_mask);
 578        }
 579
 580        hfi1_cdbg(AFFINITY,
 581                  "[%s] Completion vector affinity CPU set(s) %*pbl",
 582                  rvt_get_ibdev_name(&(dd)->verbs_dev.rdi),
 583                  cpumask_pr_args(dev_comp_vect_mask));
 584
 585        return 0;
 586
 587fail:
 588        for (j = 0; j < i; j++)
 589                per_cpu_affinity_put_max(&entry->comp_vect_mask,
 590                                         entry->comp_vect_affinity);
 591
 592        return curr_cpu;
 593}
 594
 595/*
 596 * It assumes dd->comp_vect_possible_cpus is available.
 597 */
 598static void _dev_comp_vect_cpu_mask_clean_up(struct hfi1_devdata *dd,
 599                                             struct hfi1_affinity_node *entry)
 600        __must_hold(&node_affinity.lock)
 601{
 602        int i, cpu;
 603
 604        lockdep_assert_held(&node_affinity.lock);
 605        if (!dd->comp_vect_possible_cpus)
 606                return;
 607
 608        for (i = 0; i < dd->comp_vect_possible_cpus; i++) {
 609                cpu = per_cpu_affinity_put_max(&dd->comp_vect->mask,
 610                                               entry->comp_vect_affinity);
 611                /* Clearing CPU in device completion vector cpu mask */
 612                if (cpu >= 0)
 613                        cpumask_clear_cpu(cpu, &dd->comp_vect->mask);
 614        }
 615
 616        dd->comp_vect_possible_cpus = 0;
 617}
 618
 619/*
 620 * Interrupt affinity.
 621 *
 622 * non-rcv avail gets a default mask that
 623 * starts as possible cpus with threads reset
 624 * and each rcv avail reset.
 625 *
 626 * rcv avail gets node relative 1 wrapping back
 627 * to the node relative 1 as necessary.
 628 *
 629 */
 630int hfi1_dev_affinity_init(struct hfi1_devdata *dd)
 631{
 632        int node = pcibus_to_node(dd->pcidev->bus);
 633        struct hfi1_affinity_node *entry;
 634        const struct cpumask *local_mask;
 635        int curr_cpu, possible, i, ret;
 636        bool new_entry = false;
 637
 638        /*
 639         * If the BIOS does not have the NUMA node information set, select
 640         * NUMA 0 so we get consistent performance.
 641         */
 642        if (node < 0) {
 643                dd_dev_err(dd, "Invalid PCI NUMA node. Performance may be affected\n");
 644                node = 0;
 645        }
 646        dd->node = node;
 647
 648        local_mask = cpumask_of_node(dd->node);
 649        if (cpumask_first(local_mask) >= nr_cpu_ids)
 650                local_mask = topology_core_cpumask(0);
 651
 652        mutex_lock(&node_affinity.lock);
 653        entry = node_affinity_lookup(dd->node);
 654
 655        /*
 656         * If this is the first time this NUMA node's affinity is used,
 657         * create an entry in the global affinity structure and initialize it.
 658         */
 659        if (!entry) {
 660                entry = node_affinity_allocate(node);
 661                if (!entry) {
 662                        dd_dev_err(dd,
 663                                   "Unable to allocate global affinity node\n");
 664                        ret = -ENOMEM;
 665                        goto fail;
 666                }
 667                new_entry = true;
 668
 669                init_cpu_mask_set(&entry->def_intr);
 670                init_cpu_mask_set(&entry->rcv_intr);
 671                cpumask_clear(&entry->comp_vect_mask);
 672                cpumask_clear(&entry->general_intr_mask);
 673                /* Use the "real" cpu mask of this node as the default */
 674                cpumask_and(&entry->def_intr.mask, &node_affinity.real_cpu_mask,
 675                            local_mask);
 676
 677                /* fill in the receive list */
 678                possible = cpumask_weight(&entry->def_intr.mask);
 679                curr_cpu = cpumask_first(&entry->def_intr.mask);
 680
 681                if (possible == 1) {
 682                        /* only one CPU, everyone will use it */
 683                        cpumask_set_cpu(curr_cpu, &entry->rcv_intr.mask);
 684                        cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
 685                } else {
 686                        /*
 687                         * The general/control context will be the first CPU in
 688                         * the default list, so it is removed from the default
 689                         * list and added to the general interrupt list.
 690                         */
 691                        cpumask_clear_cpu(curr_cpu, &entry->def_intr.mask);
 692                        cpumask_set_cpu(curr_cpu, &entry->general_intr_mask);
 693                        curr_cpu = cpumask_next(curr_cpu,
 694                                                &entry->def_intr.mask);
 695
 696                        /*
 697                         * Remove the remaining kernel receive queues from
 698                         * the default list and add them to the receive list.
 699                         */
 700                        for (i = 0;
 701                             i < (dd->n_krcv_queues - 1) *
 702                                  hfi1_per_node_cntr[dd->node];
 703                             i++) {
 704                                cpumask_clear_cpu(curr_cpu,
 705                                                  &entry->def_intr.mask);
 706                                cpumask_set_cpu(curr_cpu,
 707                                                &entry->rcv_intr.mask);
 708                                curr_cpu = cpumask_next(curr_cpu,
 709                                                        &entry->def_intr.mask);
 710                                if (curr_cpu >= nr_cpu_ids)
 711                                        break;
 712                        }
 713
 714                        /*
 715                         * If there ends up being 0 CPU cores leftover for SDMA
 716                         * engines, use the same CPU cores as general/control
 717                         * context.
 718                         */
 719                        if (cpumask_weight(&entry->def_intr.mask) == 0)
 720                                cpumask_copy(&entry->def_intr.mask,
 721                                             &entry->general_intr_mask);
 722                }
 723
 724                /* Determine completion vector CPUs for the entire node */
 725                cpumask_and(&entry->comp_vect_mask,
 726                            &node_affinity.real_cpu_mask, local_mask);
 727                cpumask_andnot(&entry->comp_vect_mask,
 728                               &entry->comp_vect_mask,
 729                               &entry->rcv_intr.mask);
 730                cpumask_andnot(&entry->comp_vect_mask,
 731                               &entry->comp_vect_mask,
 732                               &entry->general_intr_mask);
 733
 734                /*
 735                 * If there ends up being 0 CPU cores leftover for completion
 736                 * vectors, use the same CPU core as the general/control
 737                 * context.
 738                 */
 739                if (cpumask_weight(&entry->comp_vect_mask) == 0)
 740                        cpumask_copy(&entry->comp_vect_mask,
 741                                     &entry->general_intr_mask);
 742        }
 743
 744        ret = _dev_comp_vect_cpu_mask_init(dd, entry, new_entry);
 745        if (ret < 0)
 746                goto fail;
 747
 748        if (new_entry)
 749                node_affinity_add_tail(entry);
 750
 751        mutex_unlock(&node_affinity.lock);
 752
 753        return 0;
 754
 755fail:
 756        if (new_entry)
 757                node_affinity_destroy(entry);
 758        mutex_unlock(&node_affinity.lock);
 759        return ret;
 760}
 761
 762void hfi1_dev_affinity_clean_up(struct hfi1_devdata *dd)
 763{
 764        struct hfi1_affinity_node *entry;
 765
 766        if (dd->node < 0)
 767                return;
 768
 769        mutex_lock(&node_affinity.lock);
 770        entry = node_affinity_lookup(dd->node);
 771        if (!entry)
 772                goto unlock;
 773
 774        /*
 775         * Free device completion vector CPUs to be used by future
 776         * completion vectors
 777         */
 778        _dev_comp_vect_cpu_mask_clean_up(dd, entry);
 779unlock:
 780        mutex_unlock(&node_affinity.lock);
 781        dd->node = NUMA_NO_NODE;
 782}
 783
 784/*
 785 * Function updates the irq affinity hint for msix after it has been changed
 786 * by the user using the /proc/irq interface. This function only accepts
 787 * one cpu in the mask.
 788 */
 789static void hfi1_update_sdma_affinity(struct hfi1_msix_entry *msix, int cpu)
 790{
 791        struct sdma_engine *sde = msix->arg;
 792        struct hfi1_devdata *dd = sde->dd;
 793        struct hfi1_affinity_node *entry;
 794        struct cpu_mask_set *set;
 795        int i, old_cpu;
 796
 797        if (cpu > num_online_cpus() || cpu == sde->cpu)
 798                return;
 799
 800        mutex_lock(&node_affinity.lock);
 801        entry = node_affinity_lookup(dd->node);
 802        if (!entry)
 803                goto unlock;
 804
 805        old_cpu = sde->cpu;
 806        sde->cpu = cpu;
 807        cpumask_clear(&msix->mask);
 808        cpumask_set_cpu(cpu, &msix->mask);
 809        dd_dev_dbg(dd, "IRQ: %u, type %s engine %u -> cpu: %d\n",
 810                   msix->irq, irq_type_names[msix->type],
 811                   sde->this_idx, cpu);
 812        irq_set_affinity_hint(msix->irq, &msix->mask);
 813
 814        /*
 815         * Set the new cpu in the hfi1_affinity_node and clean
 816         * the old cpu if it is not used by any other IRQ
 817         */
 818        set = &entry->def_intr;
 819        cpumask_set_cpu(cpu, &set->mask);
 820        cpumask_set_cpu(cpu, &set->used);
 821        for (i = 0; i < dd->msix_info.max_requested; i++) {
 822                struct hfi1_msix_entry *other_msix;
 823
 824                other_msix = &dd->msix_info.msix_entries[i];
 825                if (other_msix->type != IRQ_SDMA || other_msix == msix)
 826                        continue;
 827
 828                if (cpumask_test_cpu(old_cpu, &other_msix->mask))
 829                        goto unlock;
 830        }
 831        cpumask_clear_cpu(old_cpu, &set->mask);
 832        cpumask_clear_cpu(old_cpu, &set->used);
 833unlock:
 834        mutex_unlock(&node_affinity.lock);
 835}
 836
 837static void hfi1_irq_notifier_notify(struct irq_affinity_notify *notify,
 838                                     const cpumask_t *mask)
 839{
 840        int cpu = cpumask_first(mask);
 841        struct hfi1_msix_entry *msix = container_of(notify,
 842                                                    struct hfi1_msix_entry,
 843                                                    notify);
 844
 845        /* Only one CPU configuration supported currently */
 846        hfi1_update_sdma_affinity(msix, cpu);
 847}
 848
 849static void hfi1_irq_notifier_release(struct kref *ref)
 850{
 851        /*
 852         * This is required by affinity notifier. We don't have anything to
 853         * free here.
 854         */
 855}
 856
 857static void hfi1_setup_sdma_notifier(struct hfi1_msix_entry *msix)
 858{
 859        struct irq_affinity_notify *notify = &msix->notify;
 860
 861        notify->irq = msix->irq;
 862        notify->notify = hfi1_irq_notifier_notify;
 863        notify->release = hfi1_irq_notifier_release;
 864
 865        if (irq_set_affinity_notifier(notify->irq, notify))
 866                pr_err("Failed to register sdma irq affinity notifier for irq %d\n",
 867                       notify->irq);
 868}
 869
 870static void hfi1_cleanup_sdma_notifier(struct hfi1_msix_entry *msix)
 871{
 872        struct irq_affinity_notify *notify = &msix->notify;
 873
 874        if (irq_set_affinity_notifier(notify->irq, NULL))
 875                pr_err("Failed to cleanup sdma irq affinity notifier for irq %d\n",
 876                       notify->irq);
 877}
 878
 879/*
 880 * Function sets the irq affinity for msix.
 881 * It *must* be called with node_affinity.lock held.
 882 */
 883static int get_irq_affinity(struct hfi1_devdata *dd,
 884                            struct hfi1_msix_entry *msix)
 885{
 886        cpumask_var_t diff;
 887        struct hfi1_affinity_node *entry;
 888        struct cpu_mask_set *set = NULL;
 889        struct sdma_engine *sde = NULL;
 890        struct hfi1_ctxtdata *rcd = NULL;
 891        char extra[64];
 892        int cpu = -1;
 893
 894        extra[0] = '\0';
 895        cpumask_clear(&msix->mask);
 896
 897        entry = node_affinity_lookup(dd->node);
 898
 899        switch (msix->type) {
 900        case IRQ_SDMA:
 901                sde = (struct sdma_engine *)msix->arg;
 902                scnprintf(extra, 64, "engine %u", sde->this_idx);
 903                set = &entry->def_intr;
 904                break;
 905        case IRQ_GENERAL:
 906                cpu = cpumask_first(&entry->general_intr_mask);
 907                break;
 908        case IRQ_RCVCTXT:
 909                rcd = (struct hfi1_ctxtdata *)msix->arg;
 910                if (rcd->ctxt == HFI1_CTRL_CTXT)
 911                        cpu = cpumask_first(&entry->general_intr_mask);
 912                else
 913                        set = &entry->rcv_intr;
 914                scnprintf(extra, 64, "ctxt %u", rcd->ctxt);
 915                break;
 916        default:
 917                dd_dev_err(dd, "Invalid IRQ type %d\n", msix->type);
 918                return -EINVAL;
 919        }
 920
 921        /*
 922         * The general and control contexts are placed on a particular
 923         * CPU, which is set above. Skip accounting for it. Everything else
 924         * finds its CPU here.
 925         */
 926        if (cpu == -1 && set) {
 927                if (!zalloc_cpumask_var(&diff, GFP_KERNEL))
 928                        return -ENOMEM;
 929
 930                cpu = cpu_mask_set_get_first(set, diff);
 931                if (cpu < 0) {
 932                        free_cpumask_var(diff);
 933                        dd_dev_err(dd, "Failure to obtain CPU for IRQ\n");
 934                        return cpu;
 935                }
 936
 937                free_cpumask_var(diff);
 938        }
 939
 940        cpumask_set_cpu(cpu, &msix->mask);
 941        dd_dev_info(dd, "IRQ: %u, type %s %s -> cpu: %d\n",
 942                    msix->irq, irq_type_names[msix->type],
 943                    extra, cpu);
 944        irq_set_affinity_hint(msix->irq, &msix->mask);
 945
 946        if (msix->type == IRQ_SDMA) {
 947                sde->cpu = cpu;
 948                hfi1_setup_sdma_notifier(msix);
 949        }
 950
 951        return 0;
 952}
 953
 954int hfi1_get_irq_affinity(struct hfi1_devdata *dd, struct hfi1_msix_entry *msix)
 955{
 956        int ret;
 957
 958        mutex_lock(&node_affinity.lock);
 959        ret = get_irq_affinity(dd, msix);
 960        mutex_unlock(&node_affinity.lock);
 961        return ret;
 962}
 963
 964void hfi1_put_irq_affinity(struct hfi1_devdata *dd,
 965                           struct hfi1_msix_entry *msix)
 966{
 967        struct cpu_mask_set *set = NULL;
 968        struct hfi1_ctxtdata *rcd;
 969        struct hfi1_affinity_node *entry;
 970
 971        mutex_lock(&node_affinity.lock);
 972        entry = node_affinity_lookup(dd->node);
 973
 974        switch (msix->type) {
 975        case IRQ_SDMA:
 976                set = &entry->def_intr;
 977                hfi1_cleanup_sdma_notifier(msix);
 978                break;
 979        case IRQ_GENERAL:
 980                /* Don't do accounting for general contexts */
 981                break;
 982        case IRQ_RCVCTXT:
 983                rcd = (struct hfi1_ctxtdata *)msix->arg;
 984                /* Don't do accounting for control contexts */
 985                if (rcd->ctxt != HFI1_CTRL_CTXT)
 986                        set = &entry->rcv_intr;
 987                break;
 988        default:
 989                mutex_unlock(&node_affinity.lock);
 990                return;
 991        }
 992
 993        if (set) {
 994                cpumask_andnot(&set->used, &set->used, &msix->mask);
 995                _cpu_mask_set_gen_dec(set);
 996        }
 997
 998        irq_set_affinity_hint(msix->irq, NULL);
 999        cpumask_clear(&msix->mask);
1000        mutex_unlock(&node_affinity.lock);
1001}
1002
1003/* This should be called with node_affinity.lock held */
1004static void find_hw_thread_mask(uint hw_thread_no, cpumask_var_t hw_thread_mask,
1005                                struct hfi1_affinity_node_list *affinity)
1006{
1007        int possible, curr_cpu, i;
1008        uint num_cores_per_socket = node_affinity.num_online_cpus /
1009                                        affinity->num_core_siblings /
1010                                                node_affinity.num_online_nodes;
1011
1012        cpumask_copy(hw_thread_mask, &affinity->proc.mask);
1013        if (affinity->num_core_siblings > 0) {
1014                /* Removing other siblings not needed for now */
1015                possible = cpumask_weight(hw_thread_mask);
1016                curr_cpu = cpumask_first(hw_thread_mask);
1017                for (i = 0;
1018                     i < num_cores_per_socket * node_affinity.num_online_nodes;
1019                     i++)
1020                        curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1021
1022                for (; i < possible; i++) {
1023                        cpumask_clear_cpu(curr_cpu, hw_thread_mask);
1024                        curr_cpu = cpumask_next(curr_cpu, hw_thread_mask);
1025                }
1026
1027                /* Identifying correct HW threads within physical cores */
1028                cpumask_shift_left(hw_thread_mask, hw_thread_mask,
1029                                   num_cores_per_socket *
1030                                   node_affinity.num_online_nodes *
1031                                   hw_thread_no);
1032        }
1033}
1034
1035int hfi1_get_proc_affinity(int node)
1036{
1037        int cpu = -1, ret, i;
1038        struct hfi1_affinity_node *entry;
1039        cpumask_var_t diff, hw_thread_mask, available_mask, intrs_mask;
1040        const struct cpumask *node_mask,
1041                *proc_mask = &current->cpus_allowed;
1042        struct hfi1_affinity_node_list *affinity = &node_affinity;
1043        struct cpu_mask_set *set = &affinity->proc;
1044
1045        /*
1046         * check whether process/context affinity has already
1047         * been set
1048         */
1049        if (cpumask_weight(proc_mask) == 1) {
1050                hfi1_cdbg(PROC, "PID %u %s affinity set to CPU %*pbl",
1051                          current->pid, current->comm,
1052                          cpumask_pr_args(proc_mask));
1053                /*
1054                 * Mark the pre-set CPU as used. This is atomic so we don't
1055                 * need the lock
1056                 */
1057                cpu = cpumask_first(proc_mask);
1058                cpumask_set_cpu(cpu, &set->used);
1059                goto done;
1060        } else if (cpumask_weight(proc_mask) < cpumask_weight(&set->mask)) {
1061                hfi1_cdbg(PROC, "PID %u %s affinity set to CPU set(s) %*pbl",
1062                          current->pid, current->comm,
1063                          cpumask_pr_args(proc_mask));
1064                goto done;
1065        }
1066
1067        /*
1068         * The process does not have a preset CPU affinity so find one to
1069         * recommend using the following algorithm:
1070         *
1071         * For each user process that is opening a context on HFI Y:
1072         *  a) If all cores are filled, reinitialize the bitmask
1073         *  b) Fill real cores first, then HT cores (First set of HT
1074         *     cores on all physical cores, then second set of HT core,
1075         *     and, so on) in the following order:
1076         *
1077         *     1. Same NUMA node as HFI Y and not running an IRQ
1078         *        handler
1079         *     2. Same NUMA node as HFI Y and running an IRQ handler
1080         *     3. Different NUMA node to HFI Y and not running an IRQ
1081         *        handler
1082         *     4. Different NUMA node to HFI Y and running an IRQ
1083         *        handler
1084         *  c) Mark core as filled in the bitmask. As user processes are
1085         *     done, clear cores from the bitmask.
1086         */
1087
1088        ret = zalloc_cpumask_var(&diff, GFP_KERNEL);
1089        if (!ret)
1090                goto done;
1091        ret = zalloc_cpumask_var(&hw_thread_mask, GFP_KERNEL);
1092        if (!ret)
1093                goto free_diff;
1094        ret = zalloc_cpumask_var(&available_mask, GFP_KERNEL);
1095        if (!ret)
1096                goto free_hw_thread_mask;
1097        ret = zalloc_cpumask_var(&intrs_mask, GFP_KERNEL);
1098        if (!ret)
1099                goto free_available_mask;
1100
1101        mutex_lock(&affinity->lock);
1102        /*
1103         * If we've used all available HW threads, clear the mask and start
1104         * overloading.
1105         */
1106        _cpu_mask_set_gen_inc(set);
1107
1108        /*
1109         * If NUMA node has CPUs used by interrupt handlers, include them in the
1110         * interrupt handler mask.
1111         */
1112        entry = node_affinity_lookup(node);
1113        if (entry) {
1114                cpumask_copy(intrs_mask, (entry->def_intr.gen ?
1115                                          &entry->def_intr.mask :
1116                                          &entry->def_intr.used));
1117                cpumask_or(intrs_mask, intrs_mask, (entry->rcv_intr.gen ?
1118                                                    &entry->rcv_intr.mask :
1119                                                    &entry->rcv_intr.used));
1120                cpumask_or(intrs_mask, intrs_mask, &entry->general_intr_mask);
1121        }
1122        hfi1_cdbg(PROC, "CPUs used by interrupts: %*pbl",
1123                  cpumask_pr_args(intrs_mask));
1124
1125        cpumask_copy(hw_thread_mask, &set->mask);
1126
1127        /*
1128         * If HT cores are enabled, identify which HW threads within the
1129         * physical cores should be used.
1130         */
1131        if (affinity->num_core_siblings > 0) {
1132                for (i = 0; i < affinity->num_core_siblings; i++) {
1133                        find_hw_thread_mask(i, hw_thread_mask, affinity);
1134
1135                        /*
1136                         * If there's at least one available core for this HW
1137                         * thread number, stop looking for a core.
1138                         *
1139                         * diff will always be not empty at least once in this
1140                         * loop as the used mask gets reset when
1141                         * (set->mask == set->used) before this loop.
1142                         */
1143                        cpumask_andnot(diff, hw_thread_mask, &set->used);
1144                        if (!cpumask_empty(diff))
1145                                break;
1146                }
1147        }
1148        hfi1_cdbg(PROC, "Same available HW thread on all physical CPUs: %*pbl",
1149                  cpumask_pr_args(hw_thread_mask));
1150
1151        node_mask = cpumask_of_node(node);
1152        hfi1_cdbg(PROC, "Device on NUMA %u, CPUs %*pbl", node,
1153                  cpumask_pr_args(node_mask));
1154
1155        /* Get cpumask of available CPUs on preferred NUMA */
1156        cpumask_and(available_mask, hw_thread_mask, node_mask);
1157        cpumask_andnot(available_mask, available_mask, &set->used);
1158        hfi1_cdbg(PROC, "Available CPUs on NUMA %u: %*pbl", node,
1159                  cpumask_pr_args(available_mask));
1160
1161        /*
1162         * At first, we don't want to place processes on the same
1163         * CPUs as interrupt handlers. Then, CPUs running interrupt
1164         * handlers are used.
1165         *
1166         * 1) If diff is not empty, then there are CPUs not running
1167         *    non-interrupt handlers available, so diff gets copied
1168         *    over to available_mask.
1169         * 2) If diff is empty, then all CPUs not running interrupt
1170         *    handlers are taken, so available_mask contains all
1171         *    available CPUs running interrupt handlers.
1172         * 3) If available_mask is empty, then all CPUs on the
1173         *    preferred NUMA node are taken, so other NUMA nodes are
1174         *    used for process assignments using the same method as
1175         *    the preferred NUMA node.
1176         */
1177        cpumask_andnot(diff, available_mask, intrs_mask);
1178        if (!cpumask_empty(diff))
1179                cpumask_copy(available_mask, diff);
1180
1181        /* If we don't have CPUs on the preferred node, use other NUMA nodes */
1182        if (cpumask_empty(available_mask)) {
1183                cpumask_andnot(available_mask, hw_thread_mask, &set->used);
1184                /* Excluding preferred NUMA cores */
1185                cpumask_andnot(available_mask, available_mask, node_mask);
1186                hfi1_cdbg(PROC,
1187                          "Preferred NUMA node cores are taken, cores available in other NUMA nodes: %*pbl",
1188                          cpumask_pr_args(available_mask));
1189
1190                /*
1191                 * At first, we don't want to place processes on the same
1192                 * CPUs as interrupt handlers.
1193                 */
1194                cpumask_andnot(diff, available_mask, intrs_mask);
1195                if (!cpumask_empty(diff))
1196                        cpumask_copy(available_mask, diff);
1197        }
1198        hfi1_cdbg(PROC, "Possible CPUs for process: %*pbl",
1199                  cpumask_pr_args(available_mask));
1200
1201        cpu = cpumask_first(available_mask);
1202        if (cpu >= nr_cpu_ids) /* empty */
1203                cpu = -1;
1204        else
1205                cpumask_set_cpu(cpu, &set->used);
1206
1207        mutex_unlock(&affinity->lock);
1208        hfi1_cdbg(PROC, "Process assigned to CPU %d", cpu);
1209
1210        free_cpumask_var(intrs_mask);
1211free_available_mask:
1212        free_cpumask_var(available_mask);
1213free_hw_thread_mask:
1214        free_cpumask_var(hw_thread_mask);
1215free_diff:
1216        free_cpumask_var(diff);
1217done:
1218        return cpu;
1219}
1220
1221void hfi1_put_proc_affinity(int cpu)
1222{
1223        struct hfi1_affinity_node_list *affinity = &node_affinity;
1224        struct cpu_mask_set *set = &affinity->proc;
1225
1226        if (cpu < 0)
1227                return;
1228
1229        mutex_lock(&affinity->lock);
1230        cpu_mask_set_put(set, cpu);
1231        hfi1_cdbg(PROC, "Returning CPU %d for future process assignment", cpu);
1232        mutex_unlock(&affinity->lock);
1233}
1234