linux/lib/debugobjects.c
<<
>>
Prefs
   1/*
   2 * Generic infrastructure for lifetime debugging of objects.
   3 *
   4 * Started by Thomas Gleixner
   5 *
   6 * Copyright (C) 2008, Thomas Gleixner <tglx@linutronix.de>
   7 *
   8 * For licencing details see kernel-base/COPYING
   9 */
  10
  11#define pr_fmt(fmt) "ODEBUG: " fmt
  12
  13#include <linux/debugobjects.h>
  14#include <linux/interrupt.h>
  15#include <linux/sched.h>
  16#include <linux/sched/task_stack.h>
  17#include <linux/seq_file.h>
  18#include <linux/debugfs.h>
  19#include <linux/slab.h>
  20#include <linux/hash.h>
  21#include <linux/kmemleak.h>
  22
  23#define ODEBUG_HASH_BITS        14
  24#define ODEBUG_HASH_SIZE        (1 << ODEBUG_HASH_BITS)
  25
  26#define ODEBUG_POOL_SIZE        1024
  27#define ODEBUG_POOL_MIN_LEVEL   256
  28#define ODEBUG_POOL_PERCPU_SIZE 64
  29#define ODEBUG_BATCH_SIZE       16
  30
  31#define ODEBUG_CHUNK_SHIFT      PAGE_SHIFT
  32#define ODEBUG_CHUNK_SIZE       (1 << ODEBUG_CHUNK_SHIFT)
  33#define ODEBUG_CHUNK_MASK       (~(ODEBUG_CHUNK_SIZE - 1))
  34
  35/*
  36 * We limit the freeing of debug objects via workqueue at a maximum
  37 * frequency of 10Hz and about 1024 objects for each freeing operation.
  38 * So it is freeing at most 10k debug objects per second.
  39 */
  40#define ODEBUG_FREE_WORK_MAX    1024
  41#define ODEBUG_FREE_WORK_DELAY  DIV_ROUND_UP(HZ, 10)
  42
  43struct debug_bucket {
  44        struct hlist_head       list;
  45        raw_spinlock_t          lock;
  46};
  47
  48/*
  49 * Debug object percpu free list
  50 * Access is protected by disabling irq
  51 */
  52struct debug_percpu_free {
  53        struct hlist_head       free_objs;
  54        int                     obj_free;
  55};
  56
  57static DEFINE_PER_CPU(struct debug_percpu_free, percpu_obj_pool);
  58
  59static struct debug_bucket      obj_hash[ODEBUG_HASH_SIZE];
  60
  61static struct debug_obj         obj_static_pool[ODEBUG_POOL_SIZE] __initdata;
  62
  63static DEFINE_RAW_SPINLOCK(pool_lock);
  64
  65static HLIST_HEAD(obj_pool);
  66static HLIST_HEAD(obj_to_free);
  67
  68/*
  69 * Because of the presence of percpu free pools, obj_pool_free will
  70 * under-count those in the percpu free pools. Similarly, obj_pool_used
  71 * will over-count those in the percpu free pools. Adjustments will be
  72 * made at debug_stats_show(). Both obj_pool_min_free and obj_pool_max_used
  73 * can be off.
  74 */
  75static int                      obj_pool_min_free = ODEBUG_POOL_SIZE;
  76static int                      obj_pool_free = ODEBUG_POOL_SIZE;
  77static int                      obj_pool_used;
  78static int                      obj_pool_max_used;
  79static bool                     obj_freeing;
  80/* The number of objs on the global free list */
  81static int                      obj_nr_tofree;
  82
  83static int                      debug_objects_maxchain __read_mostly;
  84static int __maybe_unused       debug_objects_maxchecked __read_mostly;
  85static int                      debug_objects_fixups __read_mostly;
  86static int                      debug_objects_warnings __read_mostly;
  87static int                      debug_objects_enabled __read_mostly
  88                                = CONFIG_DEBUG_OBJECTS_ENABLE_DEFAULT;
  89static int                      debug_objects_pool_size __read_mostly
  90                                = ODEBUG_POOL_SIZE;
  91static int                      debug_objects_pool_min_level __read_mostly
  92                                = ODEBUG_POOL_MIN_LEVEL;
  93static struct debug_obj_descr   *descr_test  __read_mostly;
  94static struct kmem_cache        *obj_cache __read_mostly;
  95
  96/*
  97 * Track numbers of kmem_cache_alloc()/free() calls done.
  98 */
  99static int                      debug_objects_allocated;
 100static int                      debug_objects_freed;
 101
 102static void free_obj_work(struct work_struct *work);
 103static DECLARE_DELAYED_WORK(debug_obj_work, free_obj_work);
 104
 105static int __init enable_object_debug(char *str)
 106{
 107        debug_objects_enabled = 1;
 108        return 0;
 109}
 110
 111static int __init disable_object_debug(char *str)
 112{
 113        debug_objects_enabled = 0;
 114        return 0;
 115}
 116
 117early_param("debug_objects", enable_object_debug);
 118early_param("no_debug_objects", disable_object_debug);
 119
 120static const char *obj_states[ODEBUG_STATE_MAX] = {
 121        [ODEBUG_STATE_NONE]             = "none",
 122        [ODEBUG_STATE_INIT]             = "initialized",
 123        [ODEBUG_STATE_INACTIVE]         = "inactive",
 124        [ODEBUG_STATE_ACTIVE]           = "active",
 125        [ODEBUG_STATE_DESTROYED]        = "destroyed",
 126        [ODEBUG_STATE_NOTAVAILABLE]     = "not available",
 127};
 128
 129static void fill_pool(void)
 130{
 131        gfp_t gfp = GFP_ATOMIC | __GFP_NORETRY | __GFP_NOWARN;
 132        struct debug_obj *obj;
 133        unsigned long flags;
 134
 135        if (likely(obj_pool_free >= debug_objects_pool_min_level))
 136                return;
 137
 138        /*
 139         * Reuse objs from the global free list; they will be reinitialized
 140         * when allocating.
 141         */
 142        while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
 143                raw_spin_lock_irqsave(&pool_lock, flags);
 144                /*
 145                 * Recheck with the lock held as the worker thread might have
 146                 * won the race and freed the global free list already.
 147                 */
 148                while (obj_nr_tofree && (obj_pool_free < obj_pool_min_free)) {
 149                        obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
 150                        hlist_del(&obj->node);
 151                        obj_nr_tofree--;
 152                        hlist_add_head(&obj->node, &obj_pool);
 153                        obj_pool_free++;
 154                }
 155                raw_spin_unlock_irqrestore(&pool_lock, flags);
 156        }
 157
 158        if (unlikely(!obj_cache))
 159                return;
 160
 161        while (obj_pool_free < debug_objects_pool_min_level) {
 162                struct debug_obj *new[ODEBUG_BATCH_SIZE];
 163                int cnt;
 164
 165                for (cnt = 0; cnt < ODEBUG_BATCH_SIZE; cnt++) {
 166                        new[cnt] = kmem_cache_zalloc(obj_cache, gfp);
 167                        if (!new[cnt])
 168                                break;
 169                }
 170                if (!cnt)
 171                        return;
 172
 173                raw_spin_lock_irqsave(&pool_lock, flags);
 174                while (cnt) {
 175                        hlist_add_head(&new[--cnt]->node, &obj_pool);
 176                        debug_objects_allocated++;
 177                        obj_pool_free++;
 178                }
 179                raw_spin_unlock_irqrestore(&pool_lock, flags);
 180        }
 181}
 182
 183/*
 184 * Lookup an object in the hash bucket.
 185 */
 186static struct debug_obj *lookup_object(void *addr, struct debug_bucket *b)
 187{
 188        struct debug_obj *obj;
 189        int cnt = 0;
 190
 191        hlist_for_each_entry(obj, &b->list, node) {
 192                cnt++;
 193                if (obj->object == addr)
 194                        return obj;
 195        }
 196        if (cnt > debug_objects_maxchain)
 197                debug_objects_maxchain = cnt;
 198
 199        return NULL;
 200}
 201
 202/*
 203 * Allocate a new object from the hlist
 204 */
 205static struct debug_obj *__alloc_object(struct hlist_head *list)
 206{
 207        struct debug_obj *obj = NULL;
 208
 209        if (list->first) {
 210                obj = hlist_entry(list->first, typeof(*obj), node);
 211                hlist_del(&obj->node);
 212        }
 213
 214        return obj;
 215}
 216
 217/*
 218 * Allocate a new object. If the pool is empty, switch off the debugger.
 219 * Must be called with interrupts disabled.
 220 */
 221static struct debug_obj *
 222alloc_object(void *addr, struct debug_bucket *b, struct debug_obj_descr *descr)
 223{
 224        struct debug_percpu_free *percpu_pool = this_cpu_ptr(&percpu_obj_pool);
 225        struct debug_obj *obj;
 226
 227        if (likely(obj_cache)) {
 228                obj = __alloc_object(&percpu_pool->free_objs);
 229                if (obj) {
 230                        percpu_pool->obj_free--;
 231                        goto init_obj;
 232                }
 233        }
 234
 235        raw_spin_lock(&pool_lock);
 236        obj = __alloc_object(&obj_pool);
 237        if (obj) {
 238                obj_pool_used++;
 239                obj_pool_free--;
 240
 241                /*
 242                 * Looking ahead, allocate one batch of debug objects and
 243                 * put them into the percpu free pool.
 244                 */
 245                if (likely(obj_cache)) {
 246                        int i;
 247
 248                        for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
 249                                struct debug_obj *obj2;
 250
 251                                obj2 = __alloc_object(&obj_pool);
 252                                if (!obj2)
 253                                        break;
 254                                hlist_add_head(&obj2->node,
 255                                               &percpu_pool->free_objs);
 256                                percpu_pool->obj_free++;
 257                                obj_pool_used++;
 258                                obj_pool_free--;
 259                        }
 260                }
 261
 262                if (obj_pool_used > obj_pool_max_used)
 263                        obj_pool_max_used = obj_pool_used;
 264
 265                if (obj_pool_free < obj_pool_min_free)
 266                        obj_pool_min_free = obj_pool_free;
 267        }
 268        raw_spin_unlock(&pool_lock);
 269
 270init_obj:
 271        if (obj) {
 272                obj->object = addr;
 273                obj->descr  = descr;
 274                obj->state  = ODEBUG_STATE_NONE;
 275                obj->astate = 0;
 276                hlist_add_head(&obj->node, &b->list);
 277        }
 278        return obj;
 279}
 280
 281/*
 282 * workqueue function to free objects.
 283 *
 284 * To reduce contention on the global pool_lock, the actual freeing of
 285 * debug objects will be delayed if the pool_lock is busy.
 286 */
 287static void free_obj_work(struct work_struct *work)
 288{
 289        struct hlist_node *tmp;
 290        struct debug_obj *obj;
 291        unsigned long flags;
 292        HLIST_HEAD(tofree);
 293
 294        WRITE_ONCE(obj_freeing, false);
 295        if (!raw_spin_trylock_irqsave(&pool_lock, flags))
 296                return;
 297
 298        if (obj_pool_free >= debug_objects_pool_size)
 299                goto free_objs;
 300
 301        /*
 302         * The objs on the pool list might be allocated before the work is
 303         * run, so recheck if pool list it full or not, if not fill pool
 304         * list from the global free list. As it is likely that a workload
 305         * may be gearing up to use more and more objects, don't free any
 306         * of them until the next round.
 307         */
 308        while (obj_nr_tofree && obj_pool_free < debug_objects_pool_size) {
 309                obj = hlist_entry(obj_to_free.first, typeof(*obj), node);
 310                hlist_del(&obj->node);
 311                hlist_add_head(&obj->node, &obj_pool);
 312                obj_pool_free++;
 313                obj_nr_tofree--;
 314        }
 315        raw_spin_unlock_irqrestore(&pool_lock, flags);
 316        return;
 317
 318free_objs:
 319        /*
 320         * Pool list is already full and there are still objs on the free
 321         * list. Move remaining free objs to a temporary list to free the
 322         * memory outside the pool_lock held region.
 323         */
 324        if (obj_nr_tofree) {
 325                hlist_move_list(&obj_to_free, &tofree);
 326                debug_objects_freed += obj_nr_tofree;
 327                obj_nr_tofree = 0;
 328        }
 329        raw_spin_unlock_irqrestore(&pool_lock, flags);
 330
 331        hlist_for_each_entry_safe(obj, tmp, &tofree, node) {
 332                hlist_del(&obj->node);
 333                kmem_cache_free(obj_cache, obj);
 334        }
 335}
 336
 337static void __free_object(struct debug_obj *obj)
 338{
 339        struct debug_obj *objs[ODEBUG_BATCH_SIZE];
 340        struct debug_percpu_free *percpu_pool;
 341        int lookahead_count = 0;
 342        unsigned long flags;
 343        bool work;
 344
 345        local_irq_save(flags);
 346        if (!obj_cache)
 347                goto free_to_obj_pool;
 348
 349        /*
 350         * Try to free it into the percpu pool first.
 351         */
 352        percpu_pool = this_cpu_ptr(&percpu_obj_pool);
 353        if (percpu_pool->obj_free < ODEBUG_POOL_PERCPU_SIZE) {
 354                hlist_add_head(&obj->node, &percpu_pool->free_objs);
 355                percpu_pool->obj_free++;
 356                local_irq_restore(flags);
 357                return;
 358        }
 359
 360        /*
 361         * As the percpu pool is full, look ahead and pull out a batch
 362         * of objects from the percpu pool and free them as well.
 363         */
 364        for (; lookahead_count < ODEBUG_BATCH_SIZE; lookahead_count++) {
 365                objs[lookahead_count] = __alloc_object(&percpu_pool->free_objs);
 366                if (!objs[lookahead_count])
 367                        break;
 368                percpu_pool->obj_free--;
 369        }
 370
 371free_to_obj_pool:
 372        raw_spin_lock(&pool_lock);
 373        work = (obj_pool_free > debug_objects_pool_size) && obj_cache &&
 374               (obj_nr_tofree < ODEBUG_FREE_WORK_MAX);
 375        obj_pool_used--;
 376
 377        if (work) {
 378                obj_nr_tofree++;
 379                hlist_add_head(&obj->node, &obj_to_free);
 380                if (lookahead_count) {
 381                        obj_nr_tofree += lookahead_count;
 382                        obj_pool_used -= lookahead_count;
 383                        while (lookahead_count) {
 384                                hlist_add_head(&objs[--lookahead_count]->node,
 385                                               &obj_to_free);
 386                        }
 387                }
 388
 389                if ((obj_pool_free > debug_objects_pool_size) &&
 390                    (obj_nr_tofree < ODEBUG_FREE_WORK_MAX)) {
 391                        int i;
 392
 393                        /*
 394                         * Free one more batch of objects from obj_pool.
 395                         */
 396                        for (i = 0; i < ODEBUG_BATCH_SIZE; i++) {
 397                                obj = __alloc_object(&obj_pool);
 398                                hlist_add_head(&obj->node, &obj_to_free);
 399                                obj_pool_free--;
 400                                obj_nr_tofree++;
 401                        }
 402                }
 403        } else {
 404                obj_pool_free++;
 405                hlist_add_head(&obj->node, &obj_pool);
 406                if (lookahead_count) {
 407                        obj_pool_free += lookahead_count;
 408                        obj_pool_used -= lookahead_count;
 409                        while (lookahead_count) {
 410                                hlist_add_head(&objs[--lookahead_count]->node,
 411                                               &obj_pool);
 412                        }
 413                }
 414        }
 415        raw_spin_unlock(&pool_lock);
 416        local_irq_restore(flags);
 417}
 418
 419/*
 420 * Put the object back into the pool and schedule work to free objects
 421 * if necessary.
 422 */
 423static void free_object(struct debug_obj *obj)
 424{
 425        __free_object(obj);
 426        if (!obj_freeing && obj_nr_tofree) {
 427                WRITE_ONCE(obj_freeing, true);
 428                schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
 429        }
 430}
 431
 432/*
 433 * We run out of memory. That means we probably have tons of objects
 434 * allocated.
 435 */
 436static void debug_objects_oom(void)
 437{
 438        struct debug_bucket *db = obj_hash;
 439        struct hlist_node *tmp;
 440        HLIST_HEAD(freelist);
 441        struct debug_obj *obj;
 442        unsigned long flags;
 443        int i;
 444
 445        pr_warn("Out of memory. ODEBUG disabled\n");
 446
 447        for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
 448                raw_spin_lock_irqsave(&db->lock, flags);
 449                hlist_move_list(&db->list, &freelist);
 450                raw_spin_unlock_irqrestore(&db->lock, flags);
 451
 452                /* Now free them */
 453                hlist_for_each_entry_safe(obj, tmp, &freelist, node) {
 454                        hlist_del(&obj->node);
 455                        free_object(obj);
 456                }
 457        }
 458}
 459
 460/*
 461 * We use the pfn of the address for the hash. That way we can check
 462 * for freed objects simply by checking the affected bucket.
 463 */
 464static struct debug_bucket *get_bucket(unsigned long addr)
 465{
 466        unsigned long hash;
 467
 468        hash = hash_long((addr >> ODEBUG_CHUNK_SHIFT), ODEBUG_HASH_BITS);
 469        return &obj_hash[hash];
 470}
 471
 472static void debug_print_object(struct debug_obj *obj, char *msg)
 473{
 474        struct debug_obj_descr *descr = obj->descr;
 475        static int limit;
 476
 477        if (limit < 5 && descr != descr_test) {
 478                void *hint = descr->debug_hint ?
 479                        descr->debug_hint(obj->object) : NULL;
 480                limit++;
 481                WARN(1, KERN_ERR "ODEBUG: %s %s (active state %u) "
 482                                 "object type: %s hint: %pS\n",
 483                        msg, obj_states[obj->state], obj->astate,
 484                        descr->name, hint);
 485        }
 486        debug_objects_warnings++;
 487}
 488
 489/*
 490 * Try to repair the damage, so we have a better chance to get useful
 491 * debug output.
 492 */
 493static bool
 494debug_object_fixup(bool (*fixup)(void *addr, enum debug_obj_state state),
 495                   void * addr, enum debug_obj_state state)
 496{
 497        if (fixup && fixup(addr, state)) {
 498                debug_objects_fixups++;
 499                return true;
 500        }
 501        return false;
 502}
 503
 504static void debug_object_is_on_stack(void *addr, int onstack)
 505{
 506        int is_on_stack;
 507        static int limit;
 508
 509        if (limit > 4)
 510                return;
 511
 512        is_on_stack = object_is_on_stack(addr);
 513        if (is_on_stack == onstack)
 514                return;
 515
 516        limit++;
 517        if (is_on_stack)
 518                pr_warn("object %p is on stack %p, but NOT annotated.\n", addr,
 519                         task_stack_page(current));
 520        else
 521                pr_warn("object %p is NOT on stack %p, but annotated.\n", addr,
 522                         task_stack_page(current));
 523
 524        WARN_ON(1);
 525}
 526
 527static void
 528__debug_object_init(void *addr, struct debug_obj_descr *descr, int onstack)
 529{
 530        enum debug_obj_state state;
 531        bool check_stack = false;
 532        struct debug_bucket *db;
 533        struct debug_obj *obj;
 534        unsigned long flags;
 535
 536        fill_pool();
 537
 538        db = get_bucket((unsigned long) addr);
 539
 540        raw_spin_lock_irqsave(&db->lock, flags);
 541
 542        obj = lookup_object(addr, db);
 543        if (!obj) {
 544                obj = alloc_object(addr, db, descr);
 545                if (!obj) {
 546                        debug_objects_enabled = 0;
 547                        raw_spin_unlock_irqrestore(&db->lock, flags);
 548                        debug_objects_oom();
 549                        return;
 550                }
 551                check_stack = true;
 552        }
 553
 554        switch (obj->state) {
 555        case ODEBUG_STATE_NONE:
 556        case ODEBUG_STATE_INIT:
 557        case ODEBUG_STATE_INACTIVE:
 558                obj->state = ODEBUG_STATE_INIT;
 559                break;
 560
 561        case ODEBUG_STATE_ACTIVE:
 562                state = obj->state;
 563                raw_spin_unlock_irqrestore(&db->lock, flags);
 564                debug_print_object(obj, "init");
 565                debug_object_fixup(descr->fixup_init, addr, state);
 566                return;
 567
 568        case ODEBUG_STATE_DESTROYED:
 569                raw_spin_unlock_irqrestore(&db->lock, flags);
 570                debug_print_object(obj, "init");
 571                return;
 572        default:
 573                break;
 574        }
 575
 576        raw_spin_unlock_irqrestore(&db->lock, flags);
 577        if (check_stack)
 578                debug_object_is_on_stack(addr, onstack);
 579}
 580
 581/**
 582 * debug_object_init - debug checks when an object is initialized
 583 * @addr:       address of the object
 584 * @descr:      pointer to an object specific debug description structure
 585 */
 586void debug_object_init(void *addr, struct debug_obj_descr *descr)
 587{
 588        if (!debug_objects_enabled)
 589                return;
 590
 591        __debug_object_init(addr, descr, 0);
 592}
 593EXPORT_SYMBOL_GPL(debug_object_init);
 594
 595/**
 596 * debug_object_init_on_stack - debug checks when an object on stack is
 597 *                              initialized
 598 * @addr:       address of the object
 599 * @descr:      pointer to an object specific debug description structure
 600 */
 601void debug_object_init_on_stack(void *addr, struct debug_obj_descr *descr)
 602{
 603        if (!debug_objects_enabled)
 604                return;
 605
 606        __debug_object_init(addr, descr, 1);
 607}
 608EXPORT_SYMBOL_GPL(debug_object_init_on_stack);
 609
 610/**
 611 * debug_object_activate - debug checks when an object is activated
 612 * @addr:       address of the object
 613 * @descr:      pointer to an object specific debug description structure
 614 * Returns 0 for success, -EINVAL for check failed.
 615 */
 616int debug_object_activate(void *addr, struct debug_obj_descr *descr)
 617{
 618        enum debug_obj_state state;
 619        struct debug_bucket *db;
 620        struct debug_obj *obj;
 621        unsigned long flags;
 622        int ret;
 623        struct debug_obj o = { .object = addr,
 624                               .state = ODEBUG_STATE_NOTAVAILABLE,
 625                               .descr = descr };
 626
 627        if (!debug_objects_enabled)
 628                return 0;
 629
 630        db = get_bucket((unsigned long) addr);
 631
 632        raw_spin_lock_irqsave(&db->lock, flags);
 633
 634        obj = lookup_object(addr, db);
 635        if (obj) {
 636                bool print_object = false;
 637
 638                switch (obj->state) {
 639                case ODEBUG_STATE_INIT:
 640                case ODEBUG_STATE_INACTIVE:
 641                        obj->state = ODEBUG_STATE_ACTIVE;
 642                        ret = 0;
 643                        break;
 644
 645                case ODEBUG_STATE_ACTIVE:
 646                        state = obj->state;
 647                        raw_spin_unlock_irqrestore(&db->lock, flags);
 648                        debug_print_object(obj, "activate");
 649                        ret = debug_object_fixup(descr->fixup_activate, addr, state);
 650                        return ret ? 0 : -EINVAL;
 651
 652                case ODEBUG_STATE_DESTROYED:
 653                        print_object = true;
 654                        ret = -EINVAL;
 655                        break;
 656                default:
 657                        ret = 0;
 658                        break;
 659                }
 660                raw_spin_unlock_irqrestore(&db->lock, flags);
 661                if (print_object)
 662                        debug_print_object(obj, "activate");
 663                return ret;
 664        }
 665
 666        raw_spin_unlock_irqrestore(&db->lock, flags);
 667
 668        /*
 669         * We are here when a static object is activated. We
 670         * let the type specific code confirm whether this is
 671         * true or not. if true, we just make sure that the
 672         * static object is tracked in the object tracker. If
 673         * not, this must be a bug, so we try to fix it up.
 674         */
 675        if (descr->is_static_object && descr->is_static_object(addr)) {
 676                /* track this static object */
 677                debug_object_init(addr, descr);
 678                debug_object_activate(addr, descr);
 679        } else {
 680                debug_print_object(&o, "activate");
 681                ret = debug_object_fixup(descr->fixup_activate, addr,
 682                                        ODEBUG_STATE_NOTAVAILABLE);
 683                return ret ? 0 : -EINVAL;
 684        }
 685        return 0;
 686}
 687EXPORT_SYMBOL_GPL(debug_object_activate);
 688
 689/**
 690 * debug_object_deactivate - debug checks when an object is deactivated
 691 * @addr:       address of the object
 692 * @descr:      pointer to an object specific debug description structure
 693 */
 694void debug_object_deactivate(void *addr, struct debug_obj_descr *descr)
 695{
 696        struct debug_bucket *db;
 697        struct debug_obj *obj;
 698        unsigned long flags;
 699        bool print_object = false;
 700
 701        if (!debug_objects_enabled)
 702                return;
 703
 704        db = get_bucket((unsigned long) addr);
 705
 706        raw_spin_lock_irqsave(&db->lock, flags);
 707
 708        obj = lookup_object(addr, db);
 709        if (obj) {
 710                switch (obj->state) {
 711                case ODEBUG_STATE_INIT:
 712                case ODEBUG_STATE_INACTIVE:
 713                case ODEBUG_STATE_ACTIVE:
 714                        if (!obj->astate)
 715                                obj->state = ODEBUG_STATE_INACTIVE;
 716                        else
 717                                print_object = true;
 718                        break;
 719
 720                case ODEBUG_STATE_DESTROYED:
 721                        print_object = true;
 722                        break;
 723                default:
 724                        break;
 725                }
 726        }
 727
 728        raw_spin_unlock_irqrestore(&db->lock, flags);
 729        if (!obj) {
 730                struct debug_obj o = { .object = addr,
 731                                       .state = ODEBUG_STATE_NOTAVAILABLE,
 732                                       .descr = descr };
 733
 734                debug_print_object(&o, "deactivate");
 735        } else if (print_object) {
 736                debug_print_object(obj, "deactivate");
 737        }
 738}
 739EXPORT_SYMBOL_GPL(debug_object_deactivate);
 740
 741/**
 742 * debug_object_destroy - debug checks when an object is destroyed
 743 * @addr:       address of the object
 744 * @descr:      pointer to an object specific debug description structure
 745 */
 746void debug_object_destroy(void *addr, struct debug_obj_descr *descr)
 747{
 748        enum debug_obj_state state;
 749        struct debug_bucket *db;
 750        struct debug_obj *obj;
 751        unsigned long flags;
 752        bool print_object = false;
 753
 754        if (!debug_objects_enabled)
 755                return;
 756
 757        db = get_bucket((unsigned long) addr);
 758
 759        raw_spin_lock_irqsave(&db->lock, flags);
 760
 761        obj = lookup_object(addr, db);
 762        if (!obj)
 763                goto out_unlock;
 764
 765        switch (obj->state) {
 766        case ODEBUG_STATE_NONE:
 767        case ODEBUG_STATE_INIT:
 768        case ODEBUG_STATE_INACTIVE:
 769                obj->state = ODEBUG_STATE_DESTROYED;
 770                break;
 771        case ODEBUG_STATE_ACTIVE:
 772                state = obj->state;
 773                raw_spin_unlock_irqrestore(&db->lock, flags);
 774                debug_print_object(obj, "destroy");
 775                debug_object_fixup(descr->fixup_destroy, addr, state);
 776                return;
 777
 778        case ODEBUG_STATE_DESTROYED:
 779                print_object = true;
 780                break;
 781        default:
 782                break;
 783        }
 784out_unlock:
 785        raw_spin_unlock_irqrestore(&db->lock, flags);
 786        if (print_object)
 787                debug_print_object(obj, "destroy");
 788}
 789EXPORT_SYMBOL_GPL(debug_object_destroy);
 790
 791/**
 792 * debug_object_free - debug checks when an object is freed
 793 * @addr:       address of the object
 794 * @descr:      pointer to an object specific debug description structure
 795 */
 796void debug_object_free(void *addr, struct debug_obj_descr *descr)
 797{
 798        enum debug_obj_state state;
 799        struct debug_bucket *db;
 800        struct debug_obj *obj;
 801        unsigned long flags;
 802
 803        if (!debug_objects_enabled)
 804                return;
 805
 806        db = get_bucket((unsigned long) addr);
 807
 808        raw_spin_lock_irqsave(&db->lock, flags);
 809
 810        obj = lookup_object(addr, db);
 811        if (!obj)
 812                goto out_unlock;
 813
 814        switch (obj->state) {
 815        case ODEBUG_STATE_ACTIVE:
 816                state = obj->state;
 817                raw_spin_unlock_irqrestore(&db->lock, flags);
 818                debug_print_object(obj, "free");
 819                debug_object_fixup(descr->fixup_free, addr, state);
 820                return;
 821        default:
 822                hlist_del(&obj->node);
 823                raw_spin_unlock_irqrestore(&db->lock, flags);
 824                free_object(obj);
 825                return;
 826        }
 827out_unlock:
 828        raw_spin_unlock_irqrestore(&db->lock, flags);
 829}
 830EXPORT_SYMBOL_GPL(debug_object_free);
 831
 832/**
 833 * debug_object_assert_init - debug checks when object should be init-ed
 834 * @addr:       address of the object
 835 * @descr:      pointer to an object specific debug description structure
 836 */
 837void debug_object_assert_init(void *addr, struct debug_obj_descr *descr)
 838{
 839        struct debug_bucket *db;
 840        struct debug_obj *obj;
 841        unsigned long flags;
 842
 843        if (!debug_objects_enabled)
 844                return;
 845
 846        db = get_bucket((unsigned long) addr);
 847
 848        raw_spin_lock_irqsave(&db->lock, flags);
 849
 850        obj = lookup_object(addr, db);
 851        if (!obj) {
 852                struct debug_obj o = { .object = addr,
 853                                       .state = ODEBUG_STATE_NOTAVAILABLE,
 854                                       .descr = descr };
 855
 856                raw_spin_unlock_irqrestore(&db->lock, flags);
 857                /*
 858                 * Maybe the object is static, and we let the type specific
 859                 * code confirm. Track this static object if true, else invoke
 860                 * fixup.
 861                 */
 862                if (descr->is_static_object && descr->is_static_object(addr)) {
 863                        /* Track this static object */
 864                        debug_object_init(addr, descr);
 865                } else {
 866                        debug_print_object(&o, "assert_init");
 867                        debug_object_fixup(descr->fixup_assert_init, addr,
 868                                           ODEBUG_STATE_NOTAVAILABLE);
 869                }
 870                return;
 871        }
 872
 873        raw_spin_unlock_irqrestore(&db->lock, flags);
 874}
 875EXPORT_SYMBOL_GPL(debug_object_assert_init);
 876
 877/**
 878 * debug_object_active_state - debug checks object usage state machine
 879 * @addr:       address of the object
 880 * @descr:      pointer to an object specific debug description structure
 881 * @expect:     expected state
 882 * @next:       state to move to if expected state is found
 883 */
 884void
 885debug_object_active_state(void *addr, struct debug_obj_descr *descr,
 886                          unsigned int expect, unsigned int next)
 887{
 888        struct debug_bucket *db;
 889        struct debug_obj *obj;
 890        unsigned long flags;
 891        bool print_object = false;
 892
 893        if (!debug_objects_enabled)
 894                return;
 895
 896        db = get_bucket((unsigned long) addr);
 897
 898        raw_spin_lock_irqsave(&db->lock, flags);
 899
 900        obj = lookup_object(addr, db);
 901        if (obj) {
 902                switch (obj->state) {
 903                case ODEBUG_STATE_ACTIVE:
 904                        if (obj->astate == expect)
 905                                obj->astate = next;
 906                        else
 907                                print_object = true;
 908                        break;
 909
 910                default:
 911                        print_object = true;
 912                        break;
 913                }
 914        }
 915
 916        raw_spin_unlock_irqrestore(&db->lock, flags);
 917        if (!obj) {
 918                struct debug_obj o = { .object = addr,
 919                                       .state = ODEBUG_STATE_NOTAVAILABLE,
 920                                       .descr = descr };
 921
 922                debug_print_object(&o, "active_state");
 923        } else if (print_object) {
 924                debug_print_object(obj, "active_state");
 925        }
 926}
 927EXPORT_SYMBOL_GPL(debug_object_active_state);
 928
 929#ifdef CONFIG_DEBUG_OBJECTS_FREE
 930static void __debug_check_no_obj_freed(const void *address, unsigned long size)
 931{
 932        unsigned long flags, oaddr, saddr, eaddr, paddr, chunks;
 933        struct debug_obj_descr *descr;
 934        enum debug_obj_state state;
 935        struct debug_bucket *db;
 936        struct hlist_node *tmp;
 937        struct debug_obj *obj;
 938        int cnt, objs_checked = 0;
 939
 940        saddr = (unsigned long) address;
 941        eaddr = saddr + size;
 942        paddr = saddr & ODEBUG_CHUNK_MASK;
 943        chunks = ((eaddr - paddr) + (ODEBUG_CHUNK_SIZE - 1));
 944        chunks >>= ODEBUG_CHUNK_SHIFT;
 945
 946        for (;chunks > 0; chunks--, paddr += ODEBUG_CHUNK_SIZE) {
 947                db = get_bucket(paddr);
 948
 949repeat:
 950                cnt = 0;
 951                raw_spin_lock_irqsave(&db->lock, flags);
 952                hlist_for_each_entry_safe(obj, tmp, &db->list, node) {
 953                        cnt++;
 954                        oaddr = (unsigned long) obj->object;
 955                        if (oaddr < saddr || oaddr >= eaddr)
 956                                continue;
 957
 958                        switch (obj->state) {
 959                        case ODEBUG_STATE_ACTIVE:
 960                                descr = obj->descr;
 961                                state = obj->state;
 962                                raw_spin_unlock_irqrestore(&db->lock, flags);
 963                                debug_print_object(obj, "free");
 964                                debug_object_fixup(descr->fixup_free,
 965                                                   (void *) oaddr, state);
 966                                goto repeat;
 967                        default:
 968                                hlist_del(&obj->node);
 969                                __free_object(obj);
 970                                break;
 971                        }
 972                }
 973                raw_spin_unlock_irqrestore(&db->lock, flags);
 974
 975                if (cnt > debug_objects_maxchain)
 976                        debug_objects_maxchain = cnt;
 977
 978                objs_checked += cnt;
 979        }
 980
 981        if (objs_checked > debug_objects_maxchecked)
 982                debug_objects_maxchecked = objs_checked;
 983
 984        /* Schedule work to actually kmem_cache_free() objects */
 985        if (!obj_freeing && obj_nr_tofree) {
 986                WRITE_ONCE(obj_freeing, true);
 987                schedule_delayed_work(&debug_obj_work, ODEBUG_FREE_WORK_DELAY);
 988        }
 989}
 990
 991void debug_check_no_obj_freed(const void *address, unsigned long size)
 992{
 993        if (debug_objects_enabled)
 994                __debug_check_no_obj_freed(address, size);
 995}
 996#endif
 997
 998#ifdef CONFIG_DEBUG_FS
 999
1000static int debug_stats_show(struct seq_file *m, void *v)
1001{
1002        int cpu, obj_percpu_free = 0;
1003
1004        for_each_possible_cpu(cpu)
1005                obj_percpu_free += per_cpu(percpu_obj_pool.obj_free, cpu);
1006
1007        seq_printf(m, "max_chain     :%d\n", debug_objects_maxchain);
1008        seq_printf(m, "max_checked   :%d\n", debug_objects_maxchecked);
1009        seq_printf(m, "warnings      :%d\n", debug_objects_warnings);
1010        seq_printf(m, "fixups        :%d\n", debug_objects_fixups);
1011        seq_printf(m, "pool_free     :%d\n", obj_pool_free + obj_percpu_free);
1012        seq_printf(m, "pool_pcp_free :%d\n", obj_percpu_free);
1013        seq_printf(m, "pool_min_free :%d\n", obj_pool_min_free);
1014        seq_printf(m, "pool_used     :%d\n", obj_pool_used - obj_percpu_free);
1015        seq_printf(m, "pool_max_used :%d\n", obj_pool_max_used);
1016        seq_printf(m, "on_free_list  :%d\n", obj_nr_tofree);
1017        seq_printf(m, "objs_allocated:%d\n", debug_objects_allocated);
1018        seq_printf(m, "objs_freed    :%d\n", debug_objects_freed);
1019        return 0;
1020}
1021
1022static int debug_stats_open(struct inode *inode, struct file *filp)
1023{
1024        return single_open(filp, debug_stats_show, NULL);
1025}
1026
1027static const struct file_operations debug_stats_fops = {
1028        .open           = debug_stats_open,
1029        .read           = seq_read,
1030        .llseek         = seq_lseek,
1031        .release        = single_release,
1032};
1033
1034static int __init debug_objects_init_debugfs(void)
1035{
1036        struct dentry *dbgdir;
1037
1038        if (!debug_objects_enabled)
1039                return 0;
1040
1041        dbgdir = debugfs_create_dir("debug_objects", NULL);
1042
1043        debugfs_create_file("stats", 0444, dbgdir, NULL, &debug_stats_fops);
1044
1045        return 0;
1046}
1047__initcall(debug_objects_init_debugfs);
1048
1049#else
1050static inline void debug_objects_init_debugfs(void) { }
1051#endif
1052
1053#ifdef CONFIG_DEBUG_OBJECTS_SELFTEST
1054
1055/* Random data structure for the self test */
1056struct self_test {
1057        unsigned long   dummy1[6];
1058        int             static_init;
1059        unsigned long   dummy2[3];
1060};
1061
1062static __initdata struct debug_obj_descr descr_type_test;
1063
1064static bool __init is_static_object(void *addr)
1065{
1066        struct self_test *obj = addr;
1067
1068        return obj->static_init;
1069}
1070
1071/*
1072 * fixup_init is called when:
1073 * - an active object is initialized
1074 */
1075static bool __init fixup_init(void *addr, enum debug_obj_state state)
1076{
1077        struct self_test *obj = addr;
1078
1079        switch (state) {
1080        case ODEBUG_STATE_ACTIVE:
1081                debug_object_deactivate(obj, &descr_type_test);
1082                debug_object_init(obj, &descr_type_test);
1083                return true;
1084        default:
1085                return false;
1086        }
1087}
1088
1089/*
1090 * fixup_activate is called when:
1091 * - an active object is activated
1092 * - an unknown non-static object is activated
1093 */
1094static bool __init fixup_activate(void *addr, enum debug_obj_state state)
1095{
1096        struct self_test *obj = addr;
1097
1098        switch (state) {
1099        case ODEBUG_STATE_NOTAVAILABLE:
1100                return true;
1101        case ODEBUG_STATE_ACTIVE:
1102                debug_object_deactivate(obj, &descr_type_test);
1103                debug_object_activate(obj, &descr_type_test);
1104                return true;
1105
1106        default:
1107                return false;
1108        }
1109}
1110
1111/*
1112 * fixup_destroy is called when:
1113 * - an active object is destroyed
1114 */
1115static bool __init fixup_destroy(void *addr, enum debug_obj_state state)
1116{
1117        struct self_test *obj = addr;
1118
1119        switch (state) {
1120        case ODEBUG_STATE_ACTIVE:
1121                debug_object_deactivate(obj, &descr_type_test);
1122                debug_object_destroy(obj, &descr_type_test);
1123                return true;
1124        default:
1125                return false;
1126        }
1127}
1128
1129/*
1130 * fixup_free is called when:
1131 * - an active object is freed
1132 */
1133static bool __init fixup_free(void *addr, enum debug_obj_state state)
1134{
1135        struct self_test *obj = addr;
1136
1137        switch (state) {
1138        case ODEBUG_STATE_ACTIVE:
1139                debug_object_deactivate(obj, &descr_type_test);
1140                debug_object_free(obj, &descr_type_test);
1141                return true;
1142        default:
1143                return false;
1144        }
1145}
1146
1147static int __init
1148check_results(void *addr, enum debug_obj_state state, int fixups, int warnings)
1149{
1150        struct debug_bucket *db;
1151        struct debug_obj *obj;
1152        unsigned long flags;
1153        int res = -EINVAL;
1154
1155        db = get_bucket((unsigned long) addr);
1156
1157        raw_spin_lock_irqsave(&db->lock, flags);
1158
1159        obj = lookup_object(addr, db);
1160        if (!obj && state != ODEBUG_STATE_NONE) {
1161                WARN(1, KERN_ERR "ODEBUG: selftest object not found\n");
1162                goto out;
1163        }
1164        if (obj && obj->state != state) {
1165                WARN(1, KERN_ERR "ODEBUG: selftest wrong state: %d != %d\n",
1166                       obj->state, state);
1167                goto out;
1168        }
1169        if (fixups != debug_objects_fixups) {
1170                WARN(1, KERN_ERR "ODEBUG: selftest fixups failed %d != %d\n",
1171                       fixups, debug_objects_fixups);
1172                goto out;
1173        }
1174        if (warnings != debug_objects_warnings) {
1175                WARN(1, KERN_ERR "ODEBUG: selftest warnings failed %d != %d\n",
1176                       warnings, debug_objects_warnings);
1177                goto out;
1178        }
1179        res = 0;
1180out:
1181        raw_spin_unlock_irqrestore(&db->lock, flags);
1182        if (res)
1183                debug_objects_enabled = 0;
1184        return res;
1185}
1186
1187static __initdata struct debug_obj_descr descr_type_test = {
1188        .name                   = "selftest",
1189        .is_static_object       = is_static_object,
1190        .fixup_init             = fixup_init,
1191        .fixup_activate         = fixup_activate,
1192        .fixup_destroy          = fixup_destroy,
1193        .fixup_free             = fixup_free,
1194};
1195
1196static __initdata struct self_test obj = { .static_init = 0 };
1197
1198static void __init debug_objects_selftest(void)
1199{
1200        int fixups, oldfixups, warnings, oldwarnings;
1201        unsigned long flags;
1202
1203        local_irq_save(flags);
1204
1205        fixups = oldfixups = debug_objects_fixups;
1206        warnings = oldwarnings = debug_objects_warnings;
1207        descr_test = &descr_type_test;
1208
1209        debug_object_init(&obj, &descr_type_test);
1210        if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1211                goto out;
1212        debug_object_activate(&obj, &descr_type_test);
1213        if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1214                goto out;
1215        debug_object_activate(&obj, &descr_type_test);
1216        if (check_results(&obj, ODEBUG_STATE_ACTIVE, ++fixups, ++warnings))
1217                goto out;
1218        debug_object_deactivate(&obj, &descr_type_test);
1219        if (check_results(&obj, ODEBUG_STATE_INACTIVE, fixups, warnings))
1220                goto out;
1221        debug_object_destroy(&obj, &descr_type_test);
1222        if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, warnings))
1223                goto out;
1224        debug_object_init(&obj, &descr_type_test);
1225        if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1226                goto out;
1227        debug_object_activate(&obj, &descr_type_test);
1228        if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1229                goto out;
1230        debug_object_deactivate(&obj, &descr_type_test);
1231        if (check_results(&obj, ODEBUG_STATE_DESTROYED, fixups, ++warnings))
1232                goto out;
1233        debug_object_free(&obj, &descr_type_test);
1234        if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1235                goto out;
1236
1237        obj.static_init = 1;
1238        debug_object_activate(&obj, &descr_type_test);
1239        if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1240                goto out;
1241        debug_object_init(&obj, &descr_type_test);
1242        if (check_results(&obj, ODEBUG_STATE_INIT, ++fixups, ++warnings))
1243                goto out;
1244        debug_object_free(&obj, &descr_type_test);
1245        if (check_results(&obj, ODEBUG_STATE_NONE, fixups, warnings))
1246                goto out;
1247
1248#ifdef CONFIG_DEBUG_OBJECTS_FREE
1249        debug_object_init(&obj, &descr_type_test);
1250        if (check_results(&obj, ODEBUG_STATE_INIT, fixups, warnings))
1251                goto out;
1252        debug_object_activate(&obj, &descr_type_test);
1253        if (check_results(&obj, ODEBUG_STATE_ACTIVE, fixups, warnings))
1254                goto out;
1255        __debug_check_no_obj_freed(&obj, sizeof(obj));
1256        if (check_results(&obj, ODEBUG_STATE_NONE, ++fixups, ++warnings))
1257                goto out;
1258#endif
1259        pr_info("selftest passed\n");
1260
1261out:
1262        debug_objects_fixups = oldfixups;
1263        debug_objects_warnings = oldwarnings;
1264        descr_test = NULL;
1265
1266        local_irq_restore(flags);
1267}
1268#else
1269static inline void debug_objects_selftest(void) { }
1270#endif
1271
1272/*
1273 * Called during early boot to initialize the hash buckets and link
1274 * the static object pool objects into the poll list. After this call
1275 * the object tracker is fully operational.
1276 */
1277void __init debug_objects_early_init(void)
1278{
1279        int i;
1280
1281        for (i = 0; i < ODEBUG_HASH_SIZE; i++)
1282                raw_spin_lock_init(&obj_hash[i].lock);
1283
1284        for (i = 0; i < ODEBUG_POOL_SIZE; i++)
1285                hlist_add_head(&obj_static_pool[i].node, &obj_pool);
1286}
1287
1288/*
1289 * Convert the statically allocated objects to dynamic ones:
1290 */
1291static int __init debug_objects_replace_static_objects(void)
1292{
1293        struct debug_bucket *db = obj_hash;
1294        struct hlist_node *tmp;
1295        struct debug_obj *obj, *new;
1296        HLIST_HEAD(objects);
1297        int i, cnt = 0;
1298
1299        for (i = 0; i < ODEBUG_POOL_SIZE; i++) {
1300                obj = kmem_cache_zalloc(obj_cache, GFP_KERNEL);
1301                if (!obj)
1302                        goto free;
1303                hlist_add_head(&obj->node, &objects);
1304        }
1305
1306        /*
1307         * debug_objects_mem_init() is now called early that only one CPU is up
1308         * and interrupts have been disabled, so it is safe to replace the
1309         * active object references.
1310         */
1311
1312        /* Remove the statically allocated objects from the pool */
1313        hlist_for_each_entry_safe(obj, tmp, &obj_pool, node)
1314                hlist_del(&obj->node);
1315        /* Move the allocated objects to the pool */
1316        hlist_move_list(&objects, &obj_pool);
1317
1318        /* Replace the active object references */
1319        for (i = 0; i < ODEBUG_HASH_SIZE; i++, db++) {
1320                hlist_move_list(&db->list, &objects);
1321
1322                hlist_for_each_entry(obj, &objects, node) {
1323                        new = hlist_entry(obj_pool.first, typeof(*obj), node);
1324                        hlist_del(&new->node);
1325                        /* copy object data */
1326                        *new = *obj;
1327                        hlist_add_head(&new->node, &db->list);
1328                        cnt++;
1329                }
1330        }
1331
1332        pr_debug("%d of %d active objects replaced\n",
1333                 cnt, obj_pool_used);
1334        return 0;
1335free:
1336        hlist_for_each_entry_safe(obj, tmp, &objects, node) {
1337                hlist_del(&obj->node);
1338                kmem_cache_free(obj_cache, obj);
1339        }
1340        return -ENOMEM;
1341}
1342
1343/*
1344 * Called after the kmem_caches are functional to setup a dedicated
1345 * cache pool, which has the SLAB_DEBUG_OBJECTS flag set. This flag
1346 * prevents that the debug code is called on kmem_cache_free() for the
1347 * debug tracker objects to avoid recursive calls.
1348 */
1349void __init debug_objects_mem_init(void)
1350{
1351        int cpu, extras;
1352
1353        if (!debug_objects_enabled)
1354                return;
1355
1356        /*
1357         * Initialize the percpu object pools
1358         *
1359         * Initialization is not strictly necessary, but was done for
1360         * completeness.
1361         */
1362        for_each_possible_cpu(cpu)
1363                INIT_HLIST_HEAD(&per_cpu(percpu_obj_pool.free_objs, cpu));
1364
1365        obj_cache = kmem_cache_create("debug_objects_cache",
1366                                      sizeof (struct debug_obj), 0,
1367                                      SLAB_DEBUG_OBJECTS | SLAB_NOLEAKTRACE,
1368                                      NULL);
1369
1370        if (!obj_cache || debug_objects_replace_static_objects()) {
1371                debug_objects_enabled = 0;
1372                kmem_cache_destroy(obj_cache);
1373                pr_warn("out of memory.\n");
1374        } else
1375                debug_objects_selftest();
1376
1377        /*
1378         * Increase the thresholds for allocating and freeing objects
1379         * according to the number of possible CPUs available in the system.
1380         */
1381        extras = num_possible_cpus() * ODEBUG_BATCH_SIZE;
1382        debug_objects_pool_size += extras;
1383        debug_objects_pool_min_level += extras;
1384}
1385