linux/drivers/firmware/arm_sdei.c
<<
>>
Prefs
   1// SPDX-License-Identifier: GPL-2.0
   2// Copyright (C) 2017 Arm Ltd.
   3#define pr_fmt(fmt) "sdei: " fmt
   4
   5#include <linux/acpi.h>
   6#include <linux/arm_sdei.h>
   7#include <linux/arm-smccc.h>
   8#include <linux/atomic.h>
   9#include <linux/bitops.h>
  10#include <linux/compiler.h>
  11#include <linux/cpuhotplug.h>
  12#include <linux/cpu.h>
  13#include <linux/cpu_pm.h>
  14#include <linux/errno.h>
  15#include <linux/hardirq.h>
  16#include <linux/kernel.h>
  17#include <linux/kprobes.h>
  18#include <linux/kvm_host.h>
  19#include <linux/list.h>
  20#include <linux/mutex.h>
  21#include <linux/notifier.h>
  22#include <linux/of.h>
  23#include <linux/of_platform.h>
  24#include <linux/percpu.h>
  25#include <linux/platform_device.h>
  26#include <linux/pm.h>
  27#include <linux/ptrace.h>
  28#include <linux/preempt.h>
  29#include <linux/reboot.h>
  30#include <linux/slab.h>
  31#include <linux/smp.h>
  32#include <linux/spinlock.h>
  33#include <linux/uaccess.h>
  34
  35/*
  36 * The call to use to reach the firmware.
  37 */
  38static asmlinkage void (*sdei_firmware_call)(unsigned long function_id,
  39                      unsigned long arg0, unsigned long arg1,
  40                      unsigned long arg2, unsigned long arg3,
  41                      unsigned long arg4, struct arm_smccc_res *res);
  42
  43/* entry point from firmware to arch asm code */
  44static unsigned long sdei_entry_point;
  45
  46struct sdei_event {
  47        /* These three are protected by the sdei_list_lock */
  48        struct list_head        list;
  49        bool                    reregister;
  50        bool                    reenable;
  51
  52        u32                     event_num;
  53        u8                      type;
  54        u8                      priority;
  55
  56        /* This pointer is handed to firmware as the event argument. */
  57        union {
  58                /* Shared events */
  59                struct sdei_registered_event *registered;
  60
  61                /* CPU private events */
  62                struct sdei_registered_event __percpu *private_registered;
  63        };
  64};
  65
  66/* Take the mutex for any API call or modification. Take the mutex first. */
  67static DEFINE_MUTEX(sdei_events_lock);
  68
  69/* and then hold this when modifying the list */
  70static DEFINE_SPINLOCK(sdei_list_lock);
  71static LIST_HEAD(sdei_list);
  72
  73/* Private events are registered/enabled via IPI passing one of these */
  74struct sdei_crosscall_args {
  75        struct sdei_event *event;
  76        atomic_t errors;
  77        int first_error;
  78};
  79
  80#define CROSSCALL_INIT(arg, event)      (arg.event = event, \
  81                                         arg.first_error = 0, \
  82                                         atomic_set(&arg.errors, 0))
  83
  84static inline int sdei_do_cross_call(void *fn, struct sdei_event * event)
  85{
  86        struct sdei_crosscall_args arg;
  87
  88        CROSSCALL_INIT(arg, event);
  89        on_each_cpu(fn, &arg, true);
  90
  91        return arg.first_error;
  92}
  93
  94static inline void
  95sdei_cross_call_return(struct sdei_crosscall_args *arg, int err)
  96{
  97        if (err && (atomic_inc_return(&arg->errors) == 1))
  98                arg->first_error = err;
  99}
 100
 101static int sdei_to_linux_errno(unsigned long sdei_err)
 102{
 103        switch (sdei_err) {
 104        case SDEI_NOT_SUPPORTED:
 105                return -EOPNOTSUPP;
 106        case SDEI_INVALID_PARAMETERS:
 107                return -EINVAL;
 108        case SDEI_DENIED:
 109                return -EPERM;
 110        case SDEI_PENDING:
 111                return -EINPROGRESS;
 112        case SDEI_OUT_OF_RESOURCE:
 113                return -ENOMEM;
 114        }
 115
 116        /* Not an error value ... */
 117        return sdei_err;
 118}
 119
 120/*
 121 * If x0 is any of these values, then the call failed, use sdei_to_linux_errno()
 122 * to translate.
 123 */
 124static int sdei_is_err(struct arm_smccc_res *res)
 125{
 126        switch (res->a0) {
 127        case SDEI_NOT_SUPPORTED:
 128        case SDEI_INVALID_PARAMETERS:
 129        case SDEI_DENIED:
 130        case SDEI_PENDING:
 131        case SDEI_OUT_OF_RESOURCE:
 132                return true;
 133        }
 134
 135        return false;
 136}
 137
 138static int invoke_sdei_fn(unsigned long function_id, unsigned long arg0,
 139                          unsigned long arg1, unsigned long arg2,
 140                          unsigned long arg3, unsigned long arg4,
 141                          u64 *result)
 142{
 143        int err = 0;
 144        struct arm_smccc_res res;
 145
 146        if (sdei_firmware_call) {
 147                sdei_firmware_call(function_id, arg0, arg1, arg2, arg3, arg4,
 148                                   &res);
 149                if (sdei_is_err(&res))
 150                        err = sdei_to_linux_errno(res.a0);
 151        } else {
 152                /*
 153                 * !sdei_firmware_call means we failed to probe or called
 154                 * sdei_mark_interface_broken(). -EIO is not an error returned
 155                 * by sdei_to_linux_errno() and is used to suppress messages
 156                 * from this driver.
 157                 */
 158                err = -EIO;
 159                res.a0 = SDEI_NOT_SUPPORTED;
 160        }
 161
 162        if (result)
 163                *result = res.a0;
 164
 165        return err;
 166}
 167
 168static struct sdei_event *sdei_event_find(u32 event_num)
 169{
 170        struct sdei_event *e, *found = NULL;
 171
 172        lockdep_assert_held(&sdei_events_lock);
 173
 174        spin_lock(&sdei_list_lock);
 175        list_for_each_entry(e, &sdei_list, list) {
 176                if (e->event_num == event_num) {
 177                        found = e;
 178                        break;
 179                }
 180        }
 181        spin_unlock(&sdei_list_lock);
 182
 183        return found;
 184}
 185
 186int sdei_api_event_context(u32 query, u64 *result)
 187{
 188        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_CONTEXT, query, 0, 0, 0, 0,
 189                              result);
 190}
 191NOKPROBE_SYMBOL(sdei_api_event_context);
 192
 193static int sdei_api_event_get_info(u32 event, u32 info, u64 *result)
 194{
 195        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_GET_INFO, event, info, 0,
 196                              0, 0, result);
 197}
 198
 199static struct sdei_event *sdei_event_create(u32 event_num,
 200                                            sdei_event_callback *cb,
 201                                            void *cb_arg)
 202{
 203        int err;
 204        u64 result;
 205        struct sdei_event *event;
 206        struct sdei_registered_event *reg;
 207
 208        lockdep_assert_held(&sdei_events_lock);
 209
 210        event = kzalloc(sizeof(*event), GFP_KERNEL);
 211        if (!event)
 212                return ERR_PTR(-ENOMEM);
 213
 214        INIT_LIST_HEAD(&event->list);
 215        event->event_num = event_num;
 216
 217        err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_PRIORITY,
 218                                      &result);
 219        if (err) {
 220                kfree(event);
 221                return ERR_PTR(err);
 222        }
 223        event->priority = result;
 224
 225        err = sdei_api_event_get_info(event_num, SDEI_EVENT_INFO_EV_TYPE,
 226                                      &result);
 227        if (err) {
 228                kfree(event);
 229                return ERR_PTR(err);
 230        }
 231        event->type = result;
 232
 233        if (event->type == SDEI_EVENT_TYPE_SHARED) {
 234                reg = kzalloc(sizeof(*reg), GFP_KERNEL);
 235                if (!reg) {
 236                        kfree(event);
 237                        return ERR_PTR(-ENOMEM);
 238                }
 239
 240                reg->event_num = event_num;
 241                reg->priority = event->priority;
 242
 243                reg->callback = cb;
 244                reg->callback_arg = cb_arg;
 245                event->registered = reg;
 246        } else {
 247                int cpu;
 248                struct sdei_registered_event __percpu *regs;
 249
 250                regs = alloc_percpu(struct sdei_registered_event);
 251                if (!regs) {
 252                        kfree(event);
 253                        return ERR_PTR(-ENOMEM);
 254                }
 255
 256                for_each_possible_cpu(cpu) {
 257                        reg = per_cpu_ptr(regs, cpu);
 258
 259                        reg->event_num = event->event_num;
 260                        reg->priority = event->priority;
 261                        reg->callback = cb;
 262                        reg->callback_arg = cb_arg;
 263                }
 264
 265                event->private_registered = regs;
 266        }
 267
 268        if (sdei_event_find(event_num)) {
 269                kfree(event->registered);
 270                kfree(event);
 271                event = ERR_PTR(-EBUSY);
 272        } else {
 273                spin_lock(&sdei_list_lock);
 274                list_add(&event->list, &sdei_list);
 275                spin_unlock(&sdei_list_lock);
 276        }
 277
 278        return event;
 279}
 280
 281static void sdei_event_destroy(struct sdei_event *event)
 282{
 283        lockdep_assert_held(&sdei_events_lock);
 284
 285        spin_lock(&sdei_list_lock);
 286        list_del(&event->list);
 287        spin_unlock(&sdei_list_lock);
 288
 289        if (event->type == SDEI_EVENT_TYPE_SHARED)
 290                kfree(event->registered);
 291        else
 292                free_percpu(event->private_registered);
 293
 294        kfree(event);
 295}
 296
 297static int sdei_api_get_version(u64 *version)
 298{
 299        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_VERSION, 0, 0, 0, 0, 0, version);
 300}
 301
 302int sdei_mask_local_cpu(void)
 303{
 304        int err;
 305
 306        WARN_ON_ONCE(preemptible());
 307
 308        err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_MASK, 0, 0, 0, 0, 0, NULL);
 309        if (err && err != -EIO) {
 310                pr_warn_once("failed to mask CPU[%u]: %d\n",
 311                              smp_processor_id(), err);
 312                return err;
 313        }
 314
 315        return 0;
 316}
 317
 318static void _ipi_mask_cpu(void *ignored)
 319{
 320        sdei_mask_local_cpu();
 321}
 322
 323int sdei_unmask_local_cpu(void)
 324{
 325        int err;
 326
 327        WARN_ON_ONCE(preemptible());
 328
 329        err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PE_UNMASK, 0, 0, 0, 0, 0, NULL);
 330        if (err && err != -EIO) {
 331                pr_warn_once("failed to unmask CPU[%u]: %d\n",
 332                             smp_processor_id(), err);
 333                return err;
 334        }
 335
 336        return 0;
 337}
 338
 339static void _ipi_unmask_cpu(void *ignored)
 340{
 341        sdei_unmask_local_cpu();
 342}
 343
 344static void _ipi_private_reset(void *ignored)
 345{
 346        int err;
 347
 348        err = invoke_sdei_fn(SDEI_1_0_FN_SDEI_PRIVATE_RESET, 0, 0, 0, 0, 0,
 349                             NULL);
 350        if (err && err != -EIO)
 351                pr_warn_once("failed to reset CPU[%u]: %d\n",
 352                             smp_processor_id(), err);
 353}
 354
 355static int sdei_api_shared_reset(void)
 356{
 357        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_SHARED_RESET, 0, 0, 0, 0, 0,
 358                              NULL);
 359}
 360
 361static void sdei_mark_interface_broken(void)
 362{
 363        pr_err("disabling SDEI firmware interface\n");
 364        on_each_cpu(&_ipi_mask_cpu, NULL, true);
 365        sdei_firmware_call = NULL;
 366}
 367
 368static int sdei_platform_reset(void)
 369{
 370        int err;
 371
 372        on_each_cpu(&_ipi_private_reset, NULL, true);
 373        err = sdei_api_shared_reset();
 374        if (err) {
 375                pr_err("Failed to reset platform: %d\n", err);
 376                sdei_mark_interface_broken();
 377        }
 378
 379        return err;
 380}
 381
 382static int sdei_api_event_enable(u32 event_num)
 383{
 384        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_ENABLE, event_num, 0, 0, 0,
 385                              0, NULL);
 386}
 387
 388/* Called directly by the hotplug callbacks */
 389static void _local_event_enable(void *data)
 390{
 391        int err;
 392        struct sdei_crosscall_args *arg = data;
 393
 394        WARN_ON_ONCE(preemptible());
 395
 396        err = sdei_api_event_enable(arg->event->event_num);
 397
 398        sdei_cross_call_return(arg, err);
 399}
 400
 401int sdei_event_enable(u32 event_num)
 402{
 403        int err = -EINVAL;
 404        struct sdei_event *event;
 405
 406        mutex_lock(&sdei_events_lock);
 407        event = sdei_event_find(event_num);
 408        if (!event) {
 409                mutex_unlock(&sdei_events_lock);
 410                return -ENOENT;
 411        }
 412
 413        spin_lock(&sdei_list_lock);
 414        event->reenable = true;
 415        spin_unlock(&sdei_list_lock);
 416
 417        if (event->type == SDEI_EVENT_TYPE_SHARED)
 418                err = sdei_api_event_enable(event->event_num);
 419        else
 420                err = sdei_do_cross_call(_local_event_enable, event);
 421        mutex_unlock(&sdei_events_lock);
 422
 423        return err;
 424}
 425EXPORT_SYMBOL(sdei_event_enable);
 426
 427static int sdei_api_event_disable(u32 event_num)
 428{
 429        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_DISABLE, event_num, 0, 0,
 430                              0, 0, NULL);
 431}
 432
 433static void _ipi_event_disable(void *data)
 434{
 435        int err;
 436        struct sdei_crosscall_args *arg = data;
 437
 438        err = sdei_api_event_disable(arg->event->event_num);
 439
 440        sdei_cross_call_return(arg, err);
 441}
 442
 443int sdei_event_disable(u32 event_num)
 444{
 445        int err = -EINVAL;
 446        struct sdei_event *event;
 447
 448        mutex_lock(&sdei_events_lock);
 449        event = sdei_event_find(event_num);
 450        if (!event) {
 451                mutex_unlock(&sdei_events_lock);
 452                return -ENOENT;
 453        }
 454
 455        spin_lock(&sdei_list_lock);
 456        event->reenable = false;
 457        spin_unlock(&sdei_list_lock);
 458
 459        if (event->type == SDEI_EVENT_TYPE_SHARED)
 460                err = sdei_api_event_disable(event->event_num);
 461        else
 462                err = sdei_do_cross_call(_ipi_event_disable, event);
 463        mutex_unlock(&sdei_events_lock);
 464
 465        return err;
 466}
 467EXPORT_SYMBOL(sdei_event_disable);
 468
 469static int sdei_api_event_unregister(u32 event_num)
 470{
 471        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_UNREGISTER, event_num, 0,
 472                              0, 0, 0, NULL);
 473}
 474
 475/* Called directly by the hotplug callbacks */
 476static void _local_event_unregister(void *data)
 477{
 478        int err;
 479        struct sdei_crosscall_args *arg = data;
 480
 481        WARN_ON_ONCE(preemptible());
 482
 483        err = sdei_api_event_unregister(arg->event->event_num);
 484
 485        sdei_cross_call_return(arg, err);
 486}
 487
 488static int _sdei_event_unregister(struct sdei_event *event)
 489{
 490        lockdep_assert_held(&sdei_events_lock);
 491
 492        spin_lock(&sdei_list_lock);
 493        event->reregister = false;
 494        event->reenable = false;
 495        spin_unlock(&sdei_list_lock);
 496
 497        if (event->type == SDEI_EVENT_TYPE_SHARED)
 498                return sdei_api_event_unregister(event->event_num);
 499
 500        return sdei_do_cross_call(_local_event_unregister, event);
 501}
 502
 503int sdei_event_unregister(u32 event_num)
 504{
 505        int err;
 506        struct sdei_event *event;
 507
 508        WARN_ON(in_nmi());
 509
 510        mutex_lock(&sdei_events_lock);
 511        event = sdei_event_find(event_num);
 512        do {
 513                if (!event) {
 514                        pr_warn("Event %u not registered\n", event_num);
 515                        err = -ENOENT;
 516                        break;
 517                }
 518
 519                err = _sdei_event_unregister(event);
 520                if (err)
 521                        break;
 522
 523                sdei_event_destroy(event);
 524        } while (0);
 525        mutex_unlock(&sdei_events_lock);
 526
 527        return err;
 528}
 529EXPORT_SYMBOL(sdei_event_unregister);
 530
 531/*
 532 * unregister events, but don't destroy them as they are re-registered by
 533 * sdei_reregister_shared().
 534 */
 535static int sdei_unregister_shared(void)
 536{
 537        int err = 0;
 538        struct sdei_event *event;
 539
 540        mutex_lock(&sdei_events_lock);
 541        spin_lock(&sdei_list_lock);
 542        list_for_each_entry(event, &sdei_list, list) {
 543                if (event->type != SDEI_EVENT_TYPE_SHARED)
 544                        continue;
 545
 546                err = _sdei_event_unregister(event);
 547                if (err)
 548                        break;
 549        }
 550        spin_unlock(&sdei_list_lock);
 551        mutex_unlock(&sdei_events_lock);
 552
 553        return err;
 554}
 555
 556static int sdei_api_event_register(u32 event_num, unsigned long entry_point,
 557                                   void *arg, u64 flags, u64 affinity)
 558{
 559        return invoke_sdei_fn(SDEI_1_0_FN_SDEI_EVENT_REGISTER, event_num,
 560                              (unsigned long)entry_point, (unsigned long)arg,
 561                              flags, affinity, NULL);
 562}
 563
 564/* Called directly by the hotplug callbacks */
 565static void _local_event_register(void *data)
 566{
 567        int err;
 568        struct sdei_registered_event *reg;
 569        struct sdei_crosscall_args *arg = data;
 570
 571        WARN_ON(preemptible());
 572
 573        reg = per_cpu_ptr(arg->event->private_registered, smp_processor_id());
 574        err = sdei_api_event_register(arg->event->event_num, sdei_entry_point,
 575                                      reg, 0, 0);
 576
 577        sdei_cross_call_return(arg, err);
 578}
 579
 580static int _sdei_event_register(struct sdei_event *event)
 581{
 582        int err;
 583
 584        lockdep_assert_held(&sdei_events_lock);
 585
 586        spin_lock(&sdei_list_lock);
 587        event->reregister = true;
 588        spin_unlock(&sdei_list_lock);
 589
 590        if (event->type == SDEI_EVENT_TYPE_SHARED)
 591                return sdei_api_event_register(event->event_num,
 592                                               sdei_entry_point,
 593                                               event->registered,
 594                                               SDEI_EVENT_REGISTER_RM_ANY, 0);
 595
 596
 597        err = sdei_do_cross_call(_local_event_register, event);
 598        if (err) {
 599                spin_lock(&sdei_list_lock);
 600                event->reregister = false;
 601                event->reenable = false;
 602                spin_unlock(&sdei_list_lock);
 603
 604                sdei_do_cross_call(_local_event_unregister, event);
 605        }
 606
 607        return err;
 608}
 609
 610int sdei_event_register(u32 event_num, sdei_event_callback *cb, void *arg)
 611{
 612        int err;
 613        struct sdei_event *event;
 614
 615        WARN_ON(in_nmi());
 616
 617        mutex_lock(&sdei_events_lock);
 618        do {
 619                if (sdei_event_find(event_num)) {
 620                        pr_warn("Event %u already registered\n", event_num);
 621                        err = -EBUSY;
 622                        break;
 623                }
 624
 625                event = sdei_event_create(event_num, cb, arg);
 626                if (IS_ERR(event)) {
 627                        err = PTR_ERR(event);
 628                        pr_warn("Failed to create event %u: %d\n", event_num,
 629                                err);
 630                        break;
 631                }
 632
 633                err = _sdei_event_register(event);
 634                if (err) {
 635                        sdei_event_destroy(event);
 636                        pr_warn("Failed to register event %u: %d\n", event_num,
 637                                err);
 638                }
 639        } while (0);
 640        mutex_unlock(&sdei_events_lock);
 641
 642        return err;
 643}
 644EXPORT_SYMBOL(sdei_event_register);
 645
 646static int sdei_reregister_event(struct sdei_event *event)
 647{
 648        int err;
 649
 650        lockdep_assert_held(&sdei_events_lock);
 651
 652        err = _sdei_event_register(event);
 653        if (err) {
 654                pr_err("Failed to re-register event %u\n", event->event_num);
 655                sdei_event_destroy(event);
 656                return err;
 657        }
 658
 659        if (event->reenable) {
 660                if (event->type == SDEI_EVENT_TYPE_SHARED)
 661                        err = sdei_api_event_enable(event->event_num);
 662                else
 663                        err = sdei_do_cross_call(_local_event_enable, event);
 664        }
 665
 666        if (err)
 667                pr_err("Failed to re-enable event %u\n", event->event_num);
 668
 669        return err;
 670}
 671
 672static int sdei_reregister_shared(void)
 673{
 674        int err = 0;
 675        struct sdei_event *event;
 676
 677        mutex_lock(&sdei_events_lock);
 678        spin_lock(&sdei_list_lock);
 679        list_for_each_entry(event, &sdei_list, list) {
 680                if (event->type != SDEI_EVENT_TYPE_SHARED)
 681                        continue;
 682
 683                if (event->reregister) {
 684                        err = sdei_reregister_event(event);
 685                        if (err)
 686                                break;
 687                }
 688        }
 689        spin_unlock(&sdei_list_lock);
 690        mutex_unlock(&sdei_events_lock);
 691
 692        return err;
 693}
 694
 695static int sdei_cpuhp_down(unsigned int cpu)
 696{
 697        struct sdei_event *event;
 698        struct sdei_crosscall_args arg;
 699
 700        /* un-register private events */
 701        spin_lock(&sdei_list_lock);
 702        list_for_each_entry(event, &sdei_list, list) {
 703                if (event->type == SDEI_EVENT_TYPE_SHARED)
 704                        continue;
 705
 706                CROSSCALL_INIT(arg, event);
 707                /* call the cross-call function locally... */
 708                _local_event_unregister(&arg);
 709                if (arg.first_error)
 710                        pr_err("Failed to unregister event %u: %d\n",
 711                               event->event_num, arg.first_error);
 712        }
 713        spin_unlock(&sdei_list_lock);
 714
 715        return sdei_mask_local_cpu();
 716}
 717
 718static int sdei_cpuhp_up(unsigned int cpu)
 719{
 720        struct sdei_event *event;
 721        struct sdei_crosscall_args arg;
 722
 723        /* re-register/enable private events */
 724        spin_lock(&sdei_list_lock);
 725        list_for_each_entry(event, &sdei_list, list) {
 726                if (event->type == SDEI_EVENT_TYPE_SHARED)
 727                        continue;
 728
 729                if (event->reregister) {
 730                        CROSSCALL_INIT(arg, event);
 731                        /* call the cross-call function locally... */
 732                        _local_event_register(&arg);
 733                        if (arg.first_error)
 734                                pr_err("Failed to re-register event %u: %d\n",
 735                                       event->event_num, arg.first_error);
 736                }
 737
 738                if (event->reenable) {
 739                        CROSSCALL_INIT(arg, event);
 740                        _local_event_enable(&arg);
 741                        if (arg.first_error)
 742                                pr_err("Failed to re-enable event %u: %d\n",
 743                                       event->event_num, arg.first_error);
 744                }
 745        }
 746        spin_unlock(&sdei_list_lock);
 747
 748        return sdei_unmask_local_cpu();
 749}
 750
 751/* When entering idle, mask/unmask events for this cpu */
 752static int sdei_pm_notifier(struct notifier_block *nb, unsigned long action,
 753                            void *data)
 754{
 755        int rv;
 756
 757        switch (action) {
 758        case CPU_PM_ENTER:
 759                rv = sdei_mask_local_cpu();
 760                break;
 761        case CPU_PM_EXIT:
 762        case CPU_PM_ENTER_FAILED:
 763                rv = sdei_unmask_local_cpu();
 764                break;
 765        default:
 766                return NOTIFY_DONE;
 767        }
 768
 769        if (rv)
 770                return notifier_from_errno(rv);
 771
 772        return NOTIFY_OK;
 773}
 774
 775static struct notifier_block sdei_pm_nb = {
 776        .notifier_call = sdei_pm_notifier,
 777};
 778
 779static int sdei_device_suspend(struct device *dev)
 780{
 781        on_each_cpu(_ipi_mask_cpu, NULL, true);
 782
 783        return 0;
 784}
 785
 786static int sdei_device_resume(struct device *dev)
 787{
 788        on_each_cpu(_ipi_unmask_cpu, NULL, true);
 789
 790        return 0;
 791}
 792
 793/*
 794 * We need all events to be reregistered when we resume from hibernate.
 795 *
 796 * The sequence is freeze->thaw. Reboot. freeze->restore. We unregister
 797 * events during freeze, then re-register and re-enable them during thaw
 798 * and restore.
 799 */
 800static int sdei_device_freeze(struct device *dev)
 801{
 802        int err;
 803
 804        /* unregister private events */
 805        cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 806
 807        err = sdei_unregister_shared();
 808        if (err)
 809                return err;
 810
 811        return 0;
 812}
 813
 814static int sdei_device_thaw(struct device *dev)
 815{
 816        int err;
 817
 818        /* re-register shared events */
 819        err = sdei_reregister_shared();
 820        if (err) {
 821                pr_warn("Failed to re-register shared events...\n");
 822                sdei_mark_interface_broken();
 823                return err;
 824        }
 825
 826        err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
 827                                &sdei_cpuhp_up, &sdei_cpuhp_down);
 828        if (err)
 829                pr_warn("Failed to re-register CPU hotplug notifier...\n");
 830
 831        return err;
 832}
 833
 834static int sdei_device_restore(struct device *dev)
 835{
 836        int err;
 837
 838        err = sdei_platform_reset();
 839        if (err)
 840                return err;
 841
 842        return sdei_device_thaw(dev);
 843}
 844
 845static const struct dev_pm_ops sdei_pm_ops = {
 846        .suspend = sdei_device_suspend,
 847        .resume = sdei_device_resume,
 848        .freeze = sdei_device_freeze,
 849        .thaw = sdei_device_thaw,
 850        .restore = sdei_device_restore,
 851};
 852
 853/*
 854 * Mask all CPUs and unregister all events on panic, reboot or kexec.
 855 */
 856static int sdei_reboot_notifier(struct notifier_block *nb, unsigned long action,
 857                                void *data)
 858{
 859        /*
 860         * We are going to reset the interface, after this there is no point
 861         * doing work when we take CPUs offline.
 862         */
 863        cpuhp_remove_state(CPUHP_AP_ARM_SDEI_STARTING);
 864
 865        sdei_platform_reset();
 866
 867        return NOTIFY_OK;
 868}
 869
 870static struct notifier_block sdei_reboot_nb = {
 871        .notifier_call = sdei_reboot_notifier,
 872};
 873
 874static void sdei_smccc_smc(unsigned long function_id,
 875                           unsigned long arg0, unsigned long arg1,
 876                           unsigned long arg2, unsigned long arg3,
 877                           unsigned long arg4, struct arm_smccc_res *res)
 878{
 879        arm_smccc_smc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 880}
 881
 882static void sdei_smccc_hvc(unsigned long function_id,
 883                           unsigned long arg0, unsigned long arg1,
 884                           unsigned long arg2, unsigned long arg3,
 885                           unsigned long arg4, struct arm_smccc_res *res)
 886{
 887        arm_smccc_hvc(function_id, arg0, arg1, arg2, arg3, arg4, 0, 0, res);
 888}
 889
 890static int sdei_get_conduit(struct platform_device *pdev)
 891{
 892        const char *method;
 893        struct device_node *np = pdev->dev.of_node;
 894
 895        sdei_firmware_call = NULL;
 896        if (np) {
 897                if (of_property_read_string(np, "method", &method)) {
 898                        pr_warn("missing \"method\" property\n");
 899                        return CONDUIT_INVALID;
 900                }
 901
 902                if (!strcmp("hvc", method)) {
 903                        sdei_firmware_call = &sdei_smccc_hvc;
 904                        return CONDUIT_HVC;
 905                } else if (!strcmp("smc", method)) {
 906                        sdei_firmware_call = &sdei_smccc_smc;
 907                        return CONDUIT_SMC;
 908                }
 909
 910                pr_warn("invalid \"method\" property: %s\n", method);
 911        } else if (IS_ENABLED(CONFIG_ACPI) && !acpi_disabled) {
 912                if (acpi_psci_use_hvc()) {
 913                        sdei_firmware_call = &sdei_smccc_hvc;
 914                        return CONDUIT_HVC;
 915                } else {
 916                        sdei_firmware_call = &sdei_smccc_smc;
 917                        return CONDUIT_SMC;
 918                }
 919        }
 920
 921        return CONDUIT_INVALID;
 922}
 923
 924static int sdei_probe(struct platform_device *pdev)
 925{
 926        int err;
 927        u64 ver = 0;
 928        int conduit;
 929
 930        conduit = sdei_get_conduit(pdev);
 931        if (!sdei_firmware_call)
 932                return 0;
 933
 934        err = sdei_api_get_version(&ver);
 935        if (err == -EOPNOTSUPP)
 936                pr_err("advertised but not implemented in platform firmware\n");
 937        if (err) {
 938                pr_err("Failed to get SDEI version: %d\n", err);
 939                sdei_mark_interface_broken();
 940                return err;
 941        }
 942
 943        pr_info("SDEIv%d.%d (0x%x) detected in firmware.\n",
 944                (int)SDEI_VERSION_MAJOR(ver), (int)SDEI_VERSION_MINOR(ver),
 945                (int)SDEI_VERSION_VENDOR(ver));
 946
 947        if (SDEI_VERSION_MAJOR(ver) != 1) {
 948                pr_warn("Conflicting SDEI version detected.\n");
 949                sdei_mark_interface_broken();
 950                return -EINVAL;
 951        }
 952
 953        err = sdei_platform_reset();
 954        if (err)
 955                return err;
 956
 957        sdei_entry_point = sdei_arch_get_entry_point(conduit);
 958        if (!sdei_entry_point) {
 959                /* Not supported due to hardware or boot configuration */
 960                sdei_mark_interface_broken();
 961                return 0;
 962        }
 963
 964        err = cpu_pm_register_notifier(&sdei_pm_nb);
 965        if (err) {
 966                pr_warn("Failed to register CPU PM notifier...\n");
 967                goto error;
 968        }
 969
 970        err = register_reboot_notifier(&sdei_reboot_nb);
 971        if (err) {
 972                pr_warn("Failed to register reboot notifier...\n");
 973                goto remove_cpupm;
 974        }
 975
 976        err = cpuhp_setup_state(CPUHP_AP_ARM_SDEI_STARTING, "SDEI",
 977                                &sdei_cpuhp_up, &sdei_cpuhp_down);
 978        if (err) {
 979                pr_warn("Failed to register CPU hotplug notifier...\n");
 980                goto remove_reboot;
 981        }
 982
 983        return 0;
 984
 985remove_reboot:
 986        unregister_reboot_notifier(&sdei_reboot_nb);
 987
 988remove_cpupm:
 989        cpu_pm_unregister_notifier(&sdei_pm_nb);
 990
 991error:
 992        sdei_mark_interface_broken();
 993        return err;
 994}
 995
 996static const struct of_device_id sdei_of_match[] = {
 997        { .compatible = "arm,sdei-1.0" },
 998        {}
 999};
1000
1001static struct platform_driver sdei_driver = {
1002        .driver         = {
1003                .name                   = "sdei",
1004                .pm                     = &sdei_pm_ops,
1005                .of_match_table         = sdei_of_match,
1006        },
1007        .probe          = sdei_probe,
1008};
1009
1010static bool __init sdei_present_dt(void)
1011{
1012        struct platform_device *pdev;
1013        struct device_node *np, *fw_np;
1014
1015        fw_np = of_find_node_by_name(NULL, "firmware");
1016        if (!fw_np)
1017                return false;
1018
1019        np = of_find_matching_node(fw_np, sdei_of_match);
1020        of_node_put(fw_np);
1021        if (!np)
1022                return false;
1023
1024        pdev = of_platform_device_create(np, sdei_driver.driver.name, NULL);
1025        of_node_put(np);
1026        if (!pdev)
1027                return false;
1028
1029        return true;
1030}
1031
1032static bool __init sdei_present_acpi(void)
1033{
1034        acpi_status status;
1035        struct platform_device *pdev;
1036        struct acpi_table_header *sdei_table_header;
1037
1038        if (acpi_disabled)
1039                return false;
1040
1041        status = acpi_get_table(ACPI_SIG_SDEI, 0, &sdei_table_header);
1042        if (ACPI_FAILURE(status) && status != AE_NOT_FOUND) {
1043                const char *msg = acpi_format_exception(status);
1044
1045                pr_info("Failed to get ACPI:SDEI table, %s\n", msg);
1046        }
1047        if (ACPI_FAILURE(status))
1048                return false;
1049
1050        pdev = platform_device_register_simple(sdei_driver.driver.name, 0, NULL,
1051                                               0);
1052        if (IS_ERR(pdev))
1053                return false;
1054
1055        return true;
1056}
1057
1058static int __init sdei_init(void)
1059{
1060        if (sdei_present_dt() || sdei_present_acpi())
1061                platform_driver_register(&sdei_driver);
1062
1063        return 0;
1064}
1065
1066/*
1067 * On an ACPI system SDEI needs to be ready before HEST:GHES tries to register
1068 * its events. ACPI is initialised from a subsys_initcall(), GHES is initialised
1069 * by device_initcall(). We want to be called in the middle.
1070 */
1071subsys_initcall_sync(sdei_init);
1072
1073int sdei_event_handler(struct pt_regs *regs,
1074                       struct sdei_registered_event *arg)
1075{
1076        int err;
1077        mm_segment_t orig_addr_limit;
1078        u32 event_num = arg->event_num;
1079
1080        orig_addr_limit = get_fs();
1081        set_fs(USER_DS);
1082
1083        err = arg->callback(event_num, regs, arg->callback_arg);
1084        if (err)
1085                pr_err_ratelimited("event %u on CPU %u failed with error: %d\n",
1086                                   event_num, smp_processor_id(), err);
1087
1088        set_fs(orig_addr_limit);
1089
1090        return err;
1091}
1092NOKPROBE_SYMBOL(sdei_event_handler);
1093