qemu/util/aio-posix.c
<<
>>
Prefs
   1/*
   2 * QEMU aio implementation
   3 *
   4 * Copyright IBM, Corp. 2008
   5 *
   6 * Authors:
   7 *  Anthony Liguori   <aliguori@us.ibm.com>
   8 *
   9 * This work is licensed under the terms of the GNU GPL, version 2.  See
  10 * the COPYING file in the top-level directory.
  11 *
  12 * Contributions after 2012-01-13 are licensed under the terms of the
  13 * GNU GPL, version 2 or (at your option) any later version.
  14 */
  15
  16#include "qemu/osdep.h"
  17#include "qemu-common.h"
  18#include "block/block.h"
  19#include "qemu/rcu_queue.h"
  20#include "qemu/sockets.h"
  21#include "qemu/cutils.h"
  22#include "trace.h"
  23#ifdef CONFIG_EPOLL_CREATE1
  24#include <sys/epoll.h>
  25#endif
  26
  27struct AioHandler
  28{
  29    GPollFD pfd;
  30    IOHandler *io_read;
  31    IOHandler *io_write;
  32    AioPollFn *io_poll;
  33    IOHandler *io_poll_begin;
  34    IOHandler *io_poll_end;
  35    int deleted;
  36    void *opaque;
  37    bool is_external;
  38    QLIST_ENTRY(AioHandler) node;
  39};
  40
  41#ifdef CONFIG_EPOLL_CREATE1
  42
  43/* The fd number threashold to switch to epoll */
  44#define EPOLL_ENABLE_THRESHOLD 64
  45
  46static void aio_epoll_disable(AioContext *ctx)
  47{
  48    ctx->epoll_available = false;
  49    if (!ctx->epoll_enabled) {
  50        return;
  51    }
  52    ctx->epoll_enabled = false;
  53    close(ctx->epollfd);
  54}
  55
  56static inline int epoll_events_from_pfd(int pfd_events)
  57{
  58    return (pfd_events & G_IO_IN ? EPOLLIN : 0) |
  59           (pfd_events & G_IO_OUT ? EPOLLOUT : 0) |
  60           (pfd_events & G_IO_HUP ? EPOLLHUP : 0) |
  61           (pfd_events & G_IO_ERR ? EPOLLERR : 0);
  62}
  63
  64static bool aio_epoll_try_enable(AioContext *ctx)
  65{
  66    AioHandler *node;
  67    struct epoll_event event;
  68
  69    QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
  70        int r;
  71        if (node->deleted || !node->pfd.events) {
  72            continue;
  73        }
  74        event.events = epoll_events_from_pfd(node->pfd.events);
  75        event.data.ptr = node;
  76        r = epoll_ctl(ctx->epollfd, EPOLL_CTL_ADD, node->pfd.fd, &event);
  77        if (r) {
  78            return false;
  79        }
  80    }
  81    ctx->epoll_enabled = true;
  82    return true;
  83}
  84
  85static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
  86{
  87    struct epoll_event event;
  88    int r;
  89    int ctl;
  90
  91    if (!ctx->epoll_enabled) {
  92        return;
  93    }
  94    if (!node->pfd.events) {
  95        ctl = EPOLL_CTL_DEL;
  96    } else {
  97        event.data.ptr = node;
  98        event.events = epoll_events_from_pfd(node->pfd.events);
  99        ctl = is_new ? EPOLL_CTL_ADD : EPOLL_CTL_MOD;
 100    }
 101
 102    r = epoll_ctl(ctx->epollfd, ctl, node->pfd.fd, &event);
 103    if (r) {
 104        aio_epoll_disable(ctx);
 105    }
 106}
 107
 108static int aio_epoll(AioContext *ctx, GPollFD *pfds,
 109                     unsigned npfd, int64_t timeout)
 110{
 111    AioHandler *node;
 112    int i, ret = 0;
 113    struct epoll_event events[128];
 114
 115    assert(npfd == 1);
 116    assert(pfds[0].fd == ctx->epollfd);
 117    if (timeout > 0) {
 118        ret = qemu_poll_ns(pfds, npfd, timeout);
 119    }
 120    if (timeout <= 0 || ret > 0) {
 121        ret = epoll_wait(ctx->epollfd, events,
 122                         sizeof(events) / sizeof(events[0]),
 123                         timeout);
 124        if (ret <= 0) {
 125            goto out;
 126        }
 127        for (i = 0; i < ret; i++) {
 128            int ev = events[i].events;
 129            node = events[i].data.ptr;
 130            node->pfd.revents = (ev & EPOLLIN ? G_IO_IN : 0) |
 131                (ev & EPOLLOUT ? G_IO_OUT : 0) |
 132                (ev & EPOLLHUP ? G_IO_HUP : 0) |
 133                (ev & EPOLLERR ? G_IO_ERR : 0);
 134        }
 135    }
 136out:
 137    return ret;
 138}
 139
 140static bool aio_epoll_enabled(AioContext *ctx)
 141{
 142    /* Fall back to ppoll when external clients are disabled. */
 143    return !aio_external_disabled(ctx) && ctx->epoll_enabled;
 144}
 145
 146static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
 147                                 unsigned npfd, int64_t timeout)
 148{
 149    if (!ctx->epoll_available) {
 150        return false;
 151    }
 152    if (aio_epoll_enabled(ctx)) {
 153        return true;
 154    }
 155    if (npfd >= EPOLL_ENABLE_THRESHOLD) {
 156        if (aio_epoll_try_enable(ctx)) {
 157            return true;
 158        } else {
 159            aio_epoll_disable(ctx);
 160        }
 161    }
 162    return false;
 163}
 164
 165#else
 166
 167static void aio_epoll_update(AioContext *ctx, AioHandler *node, bool is_new)
 168{
 169}
 170
 171static int aio_epoll(AioContext *ctx, GPollFD *pfds,
 172                     unsigned npfd, int64_t timeout)
 173{
 174    assert(false);
 175}
 176
 177static bool aio_epoll_enabled(AioContext *ctx)
 178{
 179    return false;
 180}
 181
 182static bool aio_epoll_check_poll(AioContext *ctx, GPollFD *pfds,
 183                          unsigned npfd, int64_t timeout)
 184{
 185    return false;
 186}
 187
 188#endif
 189
 190static AioHandler *find_aio_handler(AioContext *ctx, int fd)
 191{
 192    AioHandler *node;
 193
 194    QLIST_FOREACH(node, &ctx->aio_handlers, node) {
 195        if (node->pfd.fd == fd)
 196            if (!node->deleted)
 197                return node;
 198    }
 199
 200    return NULL;
 201}
 202
 203void aio_set_fd_handler(AioContext *ctx,
 204                        int fd,
 205                        bool is_external,
 206                        IOHandler *io_read,
 207                        IOHandler *io_write,
 208                        AioPollFn *io_poll,
 209                        void *opaque)
 210{
 211    AioHandler *node;
 212    bool is_new = false;
 213    bool deleted = false;
 214
 215    qemu_lockcnt_lock(&ctx->list_lock);
 216
 217    node = find_aio_handler(ctx, fd);
 218
 219    /* Are we deleting the fd handler? */
 220    if (!io_read && !io_write && !io_poll) {
 221        if (node == NULL) {
 222            qemu_lockcnt_unlock(&ctx->list_lock);
 223            return;
 224        }
 225
 226        /* If the GSource is in the process of being destroyed then
 227         * g_source_remove_poll() causes an assertion failure.  Skip
 228         * removal in that case, because glib cleans up its state during
 229         * destruction anyway.
 230         */
 231        if (!g_source_is_destroyed(&ctx->source)) {
 232            g_source_remove_poll(&ctx->source, &node->pfd);
 233        }
 234
 235        /* If the lock is held, just mark the node as deleted */
 236        if (qemu_lockcnt_count(&ctx->list_lock)) {
 237            node->deleted = 1;
 238            node->pfd.revents = 0;
 239        } else {
 240            /* Otherwise, delete it for real.  We can't just mark it as
 241             * deleted because deleted nodes are only cleaned up while
 242             * no one is walking the handlers list.
 243             */
 244            QLIST_REMOVE(node, node);
 245            deleted = true;
 246        }
 247
 248        if (!node->io_poll) {
 249            ctx->poll_disable_cnt--;
 250        }
 251    } else {
 252        if (node == NULL) {
 253            /* Alloc and insert if it's not already there */
 254            node = g_new0(AioHandler, 1);
 255            node->pfd.fd = fd;
 256            QLIST_INSERT_HEAD_RCU(&ctx->aio_handlers, node, node);
 257
 258            g_source_add_poll(&ctx->source, &node->pfd);
 259            is_new = true;
 260
 261            ctx->poll_disable_cnt += !io_poll;
 262        } else {
 263            ctx->poll_disable_cnt += !io_poll - !node->io_poll;
 264        }
 265
 266        /* Update handler with latest information */
 267        node->io_read = io_read;
 268        node->io_write = io_write;
 269        node->io_poll = io_poll;
 270        node->opaque = opaque;
 271        node->is_external = is_external;
 272
 273        node->pfd.events = (io_read ? G_IO_IN | G_IO_HUP | G_IO_ERR : 0);
 274        node->pfd.events |= (io_write ? G_IO_OUT | G_IO_ERR : 0);
 275    }
 276
 277    aio_epoll_update(ctx, node, is_new);
 278    qemu_lockcnt_unlock(&ctx->list_lock);
 279    aio_notify(ctx);
 280
 281    if (deleted) {
 282        g_free(node);
 283    }
 284}
 285
 286void aio_set_fd_poll(AioContext *ctx, int fd,
 287                     IOHandler *io_poll_begin,
 288                     IOHandler *io_poll_end)
 289{
 290    AioHandler *node = find_aio_handler(ctx, fd);
 291
 292    if (!node) {
 293        return;
 294    }
 295
 296    node->io_poll_begin = io_poll_begin;
 297    node->io_poll_end = io_poll_end;
 298}
 299
 300void aio_set_event_notifier(AioContext *ctx,
 301                            EventNotifier *notifier,
 302                            bool is_external,
 303                            EventNotifierHandler *io_read,
 304                            AioPollFn *io_poll)
 305{
 306    aio_set_fd_handler(ctx, event_notifier_get_fd(notifier), is_external,
 307                       (IOHandler *)io_read, NULL, io_poll, notifier);
 308}
 309
 310void aio_set_event_notifier_poll(AioContext *ctx,
 311                                 EventNotifier *notifier,
 312                                 EventNotifierHandler *io_poll_begin,
 313                                 EventNotifierHandler *io_poll_end)
 314{
 315    aio_set_fd_poll(ctx, event_notifier_get_fd(notifier),
 316                    (IOHandler *)io_poll_begin,
 317                    (IOHandler *)io_poll_end);
 318}
 319
 320static void poll_set_started(AioContext *ctx, bool started)
 321{
 322    AioHandler *node;
 323
 324    if (started == ctx->poll_started) {
 325        return;
 326    }
 327
 328    ctx->poll_started = started;
 329
 330    qemu_lockcnt_inc(&ctx->list_lock);
 331    QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
 332        IOHandler *fn;
 333
 334        if (node->deleted) {
 335            continue;
 336        }
 337
 338        if (started) {
 339            fn = node->io_poll_begin;
 340        } else {
 341            fn = node->io_poll_end;
 342        }
 343
 344        if (fn) {
 345            fn(node->opaque);
 346        }
 347    }
 348    qemu_lockcnt_dec(&ctx->list_lock);
 349}
 350
 351
 352bool aio_prepare(AioContext *ctx)
 353{
 354    /* Poll mode cannot be used with glib's event loop, disable it. */
 355    poll_set_started(ctx, false);
 356
 357    return false;
 358}
 359
 360bool aio_pending(AioContext *ctx)
 361{
 362    AioHandler *node;
 363    bool result = false;
 364
 365    /*
 366     * We have to walk very carefully in case aio_set_fd_handler is
 367     * called while we're walking.
 368     */
 369    qemu_lockcnt_inc(&ctx->list_lock);
 370
 371    QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
 372        int revents;
 373
 374        revents = node->pfd.revents & node->pfd.events;
 375        if (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR) && node->io_read &&
 376            aio_node_check(ctx, node->is_external)) {
 377            result = true;
 378            break;
 379        }
 380        if (revents & (G_IO_OUT | G_IO_ERR) && node->io_write &&
 381            aio_node_check(ctx, node->is_external)) {
 382            result = true;
 383            break;
 384        }
 385    }
 386    qemu_lockcnt_dec(&ctx->list_lock);
 387
 388    return result;
 389}
 390
 391static bool aio_dispatch_handlers(AioContext *ctx)
 392{
 393    AioHandler *node, *tmp;
 394    bool progress = false;
 395
 396    QLIST_FOREACH_SAFE_RCU(node, &ctx->aio_handlers, node, tmp) {
 397        int revents;
 398
 399        revents = node->pfd.revents & node->pfd.events;
 400        node->pfd.revents = 0;
 401
 402        if (!node->deleted &&
 403            (revents & (G_IO_IN | G_IO_HUP | G_IO_ERR)) &&
 404            aio_node_check(ctx, node->is_external) &&
 405            node->io_read) {
 406            node->io_read(node->opaque);
 407
 408            /* aio_notify() does not count as progress */
 409            if (node->opaque != &ctx->notifier) {
 410                progress = true;
 411            }
 412        }
 413        if (!node->deleted &&
 414            (revents & (G_IO_OUT | G_IO_ERR)) &&
 415            aio_node_check(ctx, node->is_external) &&
 416            node->io_write) {
 417            node->io_write(node->opaque);
 418            progress = true;
 419        }
 420
 421        if (node->deleted) {
 422            if (qemu_lockcnt_dec_if_lock(&ctx->list_lock)) {
 423                QLIST_REMOVE(node, node);
 424                g_free(node);
 425                qemu_lockcnt_inc_and_unlock(&ctx->list_lock);
 426            }
 427        }
 428    }
 429
 430    return progress;
 431}
 432
 433void aio_dispatch(AioContext *ctx)
 434{
 435    qemu_lockcnt_inc(&ctx->list_lock);
 436    aio_bh_poll(ctx);
 437    aio_dispatch_handlers(ctx);
 438    qemu_lockcnt_dec(&ctx->list_lock);
 439
 440    timerlistgroup_run_timers(&ctx->tlg);
 441}
 442
 443/* These thread-local variables are used only in a small part of aio_poll
 444 * around the call to the poll() system call.  In particular they are not
 445 * used while aio_poll is performing callbacks, which makes it much easier
 446 * to think about reentrancy!
 447 *
 448 * Stack-allocated arrays would be perfect but they have size limitations;
 449 * heap allocation is expensive enough that we want to reuse arrays across
 450 * calls to aio_poll().  And because poll() has to be called without holding
 451 * any lock, the arrays cannot be stored in AioContext.  Thread-local data
 452 * has none of the disadvantages of these three options.
 453 */
 454static __thread GPollFD *pollfds;
 455static __thread AioHandler **nodes;
 456static __thread unsigned npfd, nalloc;
 457static __thread Notifier pollfds_cleanup_notifier;
 458
 459static void pollfds_cleanup(Notifier *n, void *unused)
 460{
 461    g_assert(npfd == 0);
 462    g_free(pollfds);
 463    g_free(nodes);
 464    nalloc = 0;
 465}
 466
 467static void add_pollfd(AioHandler *node)
 468{
 469    if (npfd == nalloc) {
 470        if (nalloc == 0) {
 471            pollfds_cleanup_notifier.notify = pollfds_cleanup;
 472            qemu_thread_atexit_add(&pollfds_cleanup_notifier);
 473            nalloc = 8;
 474        } else {
 475            g_assert(nalloc <= INT_MAX);
 476            nalloc *= 2;
 477        }
 478        pollfds = g_renew(GPollFD, pollfds, nalloc);
 479        nodes = g_renew(AioHandler *, nodes, nalloc);
 480    }
 481    nodes[npfd] = node;
 482    pollfds[npfd] = (GPollFD) {
 483        .fd = node->pfd.fd,
 484        .events = node->pfd.events,
 485    };
 486    npfd++;
 487}
 488
 489static bool run_poll_handlers_once(AioContext *ctx)
 490{
 491    bool progress = false;
 492    AioHandler *node;
 493
 494    QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
 495        if (!node->deleted && node->io_poll &&
 496            aio_node_check(ctx, node->is_external) &&
 497            node->io_poll(node->opaque)) {
 498            progress = true;
 499        }
 500
 501        /* Caller handles freeing deleted nodes.  Don't do it here. */
 502    }
 503
 504    return progress;
 505}
 506
 507/* run_poll_handlers:
 508 * @ctx: the AioContext
 509 * @max_ns: maximum time to poll for, in nanoseconds
 510 *
 511 * Polls for a given time.
 512 *
 513 * Note that ctx->notify_me must be non-zero so this function can detect
 514 * aio_notify().
 515 *
 516 * Note that the caller must have incremented ctx->list_lock.
 517 *
 518 * Returns: true if progress was made, false otherwise
 519 */
 520static bool run_poll_handlers(AioContext *ctx, int64_t max_ns)
 521{
 522    bool progress;
 523    int64_t end_time;
 524
 525    assert(ctx->notify_me);
 526    assert(qemu_lockcnt_count(&ctx->list_lock) > 0);
 527    assert(ctx->poll_disable_cnt == 0);
 528
 529    trace_run_poll_handlers_begin(ctx, max_ns);
 530
 531    end_time = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) + max_ns;
 532
 533    do {
 534        progress = run_poll_handlers_once(ctx);
 535    } while (!progress && qemu_clock_get_ns(QEMU_CLOCK_REALTIME) < end_time);
 536
 537    trace_run_poll_handlers_end(ctx, progress);
 538
 539    return progress;
 540}
 541
 542/* try_poll_mode:
 543 * @ctx: the AioContext
 544 * @blocking: busy polling is only attempted when blocking is true
 545 *
 546 * ctx->notify_me must be non-zero so this function can detect aio_notify().
 547 *
 548 * Note that the caller must have incremented ctx->list_lock.
 549 *
 550 * Returns: true if progress was made, false otherwise
 551 */
 552static bool try_poll_mode(AioContext *ctx, bool blocking)
 553{
 554    if (blocking && ctx->poll_max_ns && ctx->poll_disable_cnt == 0) {
 555        /* See qemu_soonest_timeout() uint64_t hack */
 556        int64_t max_ns = MIN((uint64_t)aio_compute_timeout(ctx),
 557                             (uint64_t)ctx->poll_ns);
 558
 559        if (max_ns) {
 560            poll_set_started(ctx, true);
 561
 562            if (run_poll_handlers(ctx, max_ns)) {
 563                return true;
 564            }
 565        }
 566    }
 567
 568    poll_set_started(ctx, false);
 569
 570    /* Even if we don't run busy polling, try polling once in case it can make
 571     * progress and the caller will be able to avoid ppoll(2)/epoll_wait(2).
 572     */
 573    return run_poll_handlers_once(ctx);
 574}
 575
 576bool aio_poll(AioContext *ctx, bool blocking)
 577{
 578    AioHandler *node;
 579    int i;
 580    int ret = 0;
 581    bool progress;
 582    int64_t timeout;
 583    int64_t start = 0;
 584
 585    /* aio_notify can avoid the expensive event_notifier_set if
 586     * everything (file descriptors, bottom halves, timers) will
 587     * be re-evaluated before the next blocking poll().  This is
 588     * already true when aio_poll is called with blocking == false;
 589     * if blocking == true, it is only true after poll() returns,
 590     * so disable the optimization now.
 591     */
 592    if (blocking) {
 593        atomic_add(&ctx->notify_me, 2);
 594    }
 595
 596    qemu_lockcnt_inc(&ctx->list_lock);
 597
 598    if (ctx->poll_max_ns) {
 599        start = qemu_clock_get_ns(QEMU_CLOCK_REALTIME);
 600    }
 601
 602    progress = try_poll_mode(ctx, blocking);
 603    if (!progress) {
 604        assert(npfd == 0);
 605
 606        /* fill pollfds */
 607
 608        if (!aio_epoll_enabled(ctx)) {
 609            QLIST_FOREACH_RCU(node, &ctx->aio_handlers, node) {
 610                if (!node->deleted && node->pfd.events
 611                    && aio_node_check(ctx, node->is_external)) {
 612                    add_pollfd(node);
 613                }
 614            }
 615        }
 616
 617        timeout = blocking ? aio_compute_timeout(ctx) : 0;
 618
 619        /* wait until next event */
 620        if (aio_epoll_check_poll(ctx, pollfds, npfd, timeout)) {
 621            AioHandler epoll_handler;
 622
 623            epoll_handler.pfd.fd = ctx->epollfd;
 624            epoll_handler.pfd.events = G_IO_IN | G_IO_OUT | G_IO_HUP | G_IO_ERR;
 625            npfd = 0;
 626            add_pollfd(&epoll_handler);
 627            ret = aio_epoll(ctx, pollfds, npfd, timeout);
 628        } else  {
 629            ret = qemu_poll_ns(pollfds, npfd, timeout);
 630        }
 631    }
 632
 633    if (blocking) {
 634        atomic_sub(&ctx->notify_me, 2);
 635    }
 636
 637    /* Adjust polling time */
 638    if (ctx->poll_max_ns) {
 639        int64_t block_ns = qemu_clock_get_ns(QEMU_CLOCK_REALTIME) - start;
 640
 641        if (block_ns <= ctx->poll_ns) {
 642            /* This is the sweet spot, no adjustment needed */
 643        } else if (block_ns > ctx->poll_max_ns) {
 644            /* We'd have to poll for too long, poll less */
 645            int64_t old = ctx->poll_ns;
 646
 647            if (ctx->poll_shrink) {
 648                ctx->poll_ns /= ctx->poll_shrink;
 649            } else {
 650                ctx->poll_ns = 0;
 651            }
 652
 653            trace_poll_shrink(ctx, old, ctx->poll_ns);
 654        } else if (ctx->poll_ns < ctx->poll_max_ns &&
 655                   block_ns < ctx->poll_max_ns) {
 656            /* There is room to grow, poll longer */
 657            int64_t old = ctx->poll_ns;
 658            int64_t grow = ctx->poll_grow;
 659
 660            if (grow == 0) {
 661                grow = 2;
 662            }
 663
 664            if (ctx->poll_ns) {
 665                ctx->poll_ns *= grow;
 666            } else {
 667                ctx->poll_ns = 4000; /* start polling at 4 microseconds */
 668            }
 669
 670            if (ctx->poll_ns > ctx->poll_max_ns) {
 671                ctx->poll_ns = ctx->poll_max_ns;
 672            }
 673
 674            trace_poll_grow(ctx, old, ctx->poll_ns);
 675        }
 676    }
 677
 678    aio_notify_accept(ctx);
 679
 680    /* if we have any readable fds, dispatch event */
 681    if (ret > 0) {
 682        for (i = 0; i < npfd; i++) {
 683            nodes[i]->pfd.revents = pollfds[i].revents;
 684        }
 685    }
 686
 687    npfd = 0;
 688
 689    progress |= aio_bh_poll(ctx);
 690
 691    if (ret > 0) {
 692        progress |= aio_dispatch_handlers(ctx);
 693    }
 694
 695    qemu_lockcnt_dec(&ctx->list_lock);
 696
 697    progress |= timerlistgroup_run_timers(&ctx->tlg);
 698
 699    return progress;
 700}
 701
 702void aio_context_setup(AioContext *ctx)
 703{
 704#ifdef CONFIG_EPOLL_CREATE1
 705    assert(!ctx->epollfd);
 706    ctx->epollfd = epoll_create1(EPOLL_CLOEXEC);
 707    if (ctx->epollfd == -1) {
 708        fprintf(stderr, "Failed to create epoll instance: %s", strerror(errno));
 709        ctx->epoll_available = false;
 710    } else {
 711        ctx->epoll_available = true;
 712    }
 713#endif
 714}
 715
 716void aio_context_set_poll_params(AioContext *ctx, int64_t max_ns,
 717                                 int64_t grow, int64_t shrink, Error **errp)
 718{
 719    /* No thread synchronization here, it doesn't matter if an incorrect value
 720     * is used once.
 721     */
 722    ctx->poll_max_ns = max_ns;
 723    ctx->poll_ns = 0;
 724    ctx->poll_grow = grow;
 725    ctx->poll_shrink = shrink;
 726
 727    aio_notify(ctx);
 728}
 729