linux/mm/oom_kill.c
<<
>>
Prefs
   1/*
   2 *  linux/mm/oom_kill.c
   3 * 
   4 *  Copyright (C)  1998,2000  Rik van Riel
   5 *      Thanks go out to Claus Fischer for some serious inspiration and
   6 *      for goading me into coding this file...
   7 *  Copyright (C)  2010  Google, Inc.
   8 *      Rewritten by David Rientjes
   9 *
  10 *  The routines in this file are used to kill a process when
  11 *  we're seriously out of memory. This gets called from __alloc_pages()
  12 *  in mm/page_alloc.c when we really run out of memory.
  13 *
  14 *  Since we won't call these routines often (on a well-configured
  15 *  machine) this file will double as a 'coding guide' and a signpost
  16 *  for newbie kernel hackers. It features several pointers to major
  17 *  kernel subsystems and hints as to where to find out what things do.
  18 */
  19
  20#include <linux/oom.h>
  21#include <linux/mm.h>
  22#include <linux/err.h>
  23#include <linux/gfp.h>
  24#include <linux/sched.h>
  25#include <linux/sched/mm.h>
  26#include <linux/sched/coredump.h>
  27#include <linux/sched/task.h>
  28#include <linux/sched/debug.h>
  29#include <linux/swap.h>
  30#include <linux/timex.h>
  31#include <linux/jiffies.h>
  32#include <linux/cpuset.h>
  33#include <linux/export.h>
  34#include <linux/notifier.h>
  35#include <linux/memcontrol.h>
  36#include <linux/mempolicy.h>
  37#include <linux/security.h>
  38#include <linux/ptrace.h>
  39#include <linux/freezer.h>
  40#include <linux/ftrace.h>
  41#include <linux/ratelimit.h>
  42#include <linux/kthread.h>
  43#include <linux/init.h>
  44#include <linux/mmu_notifier.h>
  45
  46#include <asm/tlb.h>
  47#include "internal.h"
  48#include "slab.h"
  49
  50#define CREATE_TRACE_POINTS
  51#include <trace/events/oom.h>
  52
  53int sysctl_panic_on_oom;
  54int sysctl_oom_kill_allocating_task;
  55int sysctl_oom_dump_tasks = 1;
  56
  57DEFINE_MUTEX(oom_lock);
  58/* Serializes oom_score_adj and oom_score_adj_min updates */
  59DEFINE_MUTEX(oom_adj_mutex);
  60
  61static inline bool is_memcg_oom(struct oom_control *oc)
  62{
  63        return oc->memcg != NULL;
  64}
  65
  66#ifdef CONFIG_NUMA
  67/**
  68 * oom_cpuset_eligible() - check task eligiblity for kill
  69 * @start: task struct of which task to consider
  70 * @oc: pointer to struct oom_control
  71 *
  72 * Task eligibility is determined by whether or not a candidate task, @tsk,
  73 * shares the same mempolicy nodes as current if it is bound by such a policy
  74 * and whether or not it has the same set of allowed cpuset nodes.
  75 *
  76 * This function is assuming oom-killer context and 'current' has triggered
  77 * the oom-killer.
  78 */
  79static bool oom_cpuset_eligible(struct task_struct *start,
  80                                struct oom_control *oc)
  81{
  82        struct task_struct *tsk;
  83        bool ret = false;
  84        const nodemask_t *mask = oc->nodemask;
  85
  86        if (is_memcg_oom(oc))
  87                return true;
  88
  89        rcu_read_lock();
  90        for_each_thread(start, tsk) {
  91                if (mask) {
  92                        /*
  93                         * If this is a mempolicy constrained oom, tsk's
  94                         * cpuset is irrelevant.  Only return true if its
  95                         * mempolicy intersects current, otherwise it may be
  96                         * needlessly killed.
  97                         */
  98                        ret = mempolicy_nodemask_intersects(tsk, mask);
  99                } else {
 100                        /*
 101                         * This is not a mempolicy constrained oom, so only
 102                         * check the mems of tsk's cpuset.
 103                         */
 104                        ret = cpuset_mems_allowed_intersects(current, tsk);
 105                }
 106                if (ret)
 107                        break;
 108        }
 109        rcu_read_unlock();
 110
 111        return ret;
 112}
 113#else
 114static bool oom_cpuset_eligible(struct task_struct *tsk, struct oom_control *oc)
 115{
 116        return true;
 117}
 118#endif /* CONFIG_NUMA */
 119
 120/*
 121 * The process p may have detached its own ->mm while exiting or through
 122 * kthread_use_mm(), but one or more of its subthreads may still have a valid
 123 * pointer.  Return p, or any of its subthreads with a valid ->mm, with
 124 * task_lock() held.
 125 */
 126struct task_struct *find_lock_task_mm(struct task_struct *p)
 127{
 128        struct task_struct *t;
 129
 130        rcu_read_lock();
 131
 132        for_each_thread(p, t) {
 133                task_lock(t);
 134                if (likely(t->mm))
 135                        goto found;
 136                task_unlock(t);
 137        }
 138        t = NULL;
 139found:
 140        rcu_read_unlock();
 141
 142        return t;
 143}
 144
 145/*
 146 * order == -1 means the oom kill is required by sysrq, otherwise only
 147 * for display purposes.
 148 */
 149static inline bool is_sysrq_oom(struct oom_control *oc)
 150{
 151        return oc->order == -1;
 152}
 153
 154/* return true if the task is not adequate as candidate victim task. */
 155static bool oom_unkillable_task(struct task_struct *p)
 156{
 157        if (is_global_init(p))
 158                return true;
 159        if (p->flags & PF_KTHREAD)
 160                return true;
 161        return false;
 162}
 163
 164/*
 165 * Print out unreclaimble slabs info when unreclaimable slabs amount is greater
 166 * than all user memory (LRU pages)
 167 */
 168static bool is_dump_unreclaim_slabs(void)
 169{
 170        unsigned long nr_lru;
 171
 172        nr_lru = global_node_page_state(NR_ACTIVE_ANON) +
 173                 global_node_page_state(NR_INACTIVE_ANON) +
 174                 global_node_page_state(NR_ACTIVE_FILE) +
 175                 global_node_page_state(NR_INACTIVE_FILE) +
 176                 global_node_page_state(NR_ISOLATED_ANON) +
 177                 global_node_page_state(NR_ISOLATED_FILE) +
 178                 global_node_page_state(NR_UNEVICTABLE);
 179
 180        return (global_node_page_state_pages(NR_SLAB_UNRECLAIMABLE_B) > nr_lru);
 181}
 182
 183/**
 184 * oom_badness - heuristic function to determine which candidate task to kill
 185 * @p: task struct of which task we should calculate
 186 * @totalpages: total present RAM allowed for page allocation
 187 *
 188 * The heuristic for determining which task to kill is made to be as simple and
 189 * predictable as possible.  The goal is to return the highest value for the
 190 * task consuming the most memory to avoid subsequent oom failures.
 191 */
 192long oom_badness(struct task_struct *p, unsigned long totalpages)
 193{
 194        long points;
 195        long adj;
 196
 197        if (oom_unkillable_task(p))
 198                return LONG_MIN;
 199
 200        p = find_lock_task_mm(p);
 201        if (!p)
 202                return LONG_MIN;
 203
 204        /*
 205         * Do not even consider tasks which are explicitly marked oom
 206         * unkillable or have been already oom reaped or the are in
 207         * the middle of vfork
 208         */
 209        adj = (long)p->signal->oom_score_adj;
 210        if (adj == OOM_SCORE_ADJ_MIN ||
 211                        test_bit(MMF_OOM_SKIP, &p->mm->flags) ||
 212                        in_vfork(p)) {
 213                task_unlock(p);
 214                return LONG_MIN;
 215        }
 216
 217        /*
 218         * The baseline for the badness score is the proportion of RAM that each
 219         * task's rss, pagetable and swap space use.
 220         */
 221        points = get_mm_rss(p->mm) + get_mm_counter(p->mm, MM_SWAPENTS) +
 222                mm_pgtables_bytes(p->mm) / PAGE_SIZE;
 223        task_unlock(p);
 224
 225        /* Normalize to oom_score_adj units */
 226        adj *= totalpages / 1000;
 227        points += adj;
 228
 229        return points;
 230}
 231
 232static const char * const oom_constraint_text[] = {
 233        [CONSTRAINT_NONE] = "CONSTRAINT_NONE",
 234        [CONSTRAINT_CPUSET] = "CONSTRAINT_CPUSET",
 235        [CONSTRAINT_MEMORY_POLICY] = "CONSTRAINT_MEMORY_POLICY",
 236        [CONSTRAINT_MEMCG] = "CONSTRAINT_MEMCG",
 237};
 238
 239/*
 240 * Determine the type of allocation constraint.
 241 */
 242static enum oom_constraint constrained_alloc(struct oom_control *oc)
 243{
 244        struct zone *zone;
 245        struct zoneref *z;
 246        enum zone_type highest_zoneidx = gfp_zone(oc->gfp_mask);
 247        bool cpuset_limited = false;
 248        int nid;
 249
 250        if (is_memcg_oom(oc)) {
 251                oc->totalpages = mem_cgroup_get_max(oc->memcg) ?: 1;
 252                return CONSTRAINT_MEMCG;
 253        }
 254
 255        /* Default to all available memory */
 256        oc->totalpages = totalram_pages() + total_swap_pages;
 257
 258        if (!IS_ENABLED(CONFIG_NUMA))
 259                return CONSTRAINT_NONE;
 260
 261        if (!oc->zonelist)
 262                return CONSTRAINT_NONE;
 263        /*
 264         * Reach here only when __GFP_NOFAIL is used. So, we should avoid
 265         * to kill current.We have to random task kill in this case.
 266         * Hopefully, CONSTRAINT_THISNODE...but no way to handle it, now.
 267         */
 268        if (oc->gfp_mask & __GFP_THISNODE)
 269                return CONSTRAINT_NONE;
 270
 271        /*
 272         * This is not a __GFP_THISNODE allocation, so a truncated nodemask in
 273         * the page allocator means a mempolicy is in effect.  Cpuset policy
 274         * is enforced in get_page_from_freelist().
 275         */
 276        if (oc->nodemask &&
 277            !nodes_subset(node_states[N_MEMORY], *oc->nodemask)) {
 278                oc->totalpages = total_swap_pages;
 279                for_each_node_mask(nid, *oc->nodemask)
 280                        oc->totalpages += node_present_pages(nid);
 281                return CONSTRAINT_MEMORY_POLICY;
 282        }
 283
 284        /* Check this allocation failure is caused by cpuset's wall function */
 285        for_each_zone_zonelist_nodemask(zone, z, oc->zonelist,
 286                        highest_zoneidx, oc->nodemask)
 287                if (!cpuset_zone_allowed(zone, oc->gfp_mask))
 288                        cpuset_limited = true;
 289
 290        if (cpuset_limited) {
 291                oc->totalpages = total_swap_pages;
 292                for_each_node_mask(nid, cpuset_current_mems_allowed)
 293                        oc->totalpages += node_present_pages(nid);
 294                return CONSTRAINT_CPUSET;
 295        }
 296        return CONSTRAINT_NONE;
 297}
 298
 299static int oom_evaluate_task(struct task_struct *task, void *arg)
 300{
 301        struct oom_control *oc = arg;
 302        long points;
 303
 304        if (oom_unkillable_task(task))
 305                goto next;
 306
 307        /* p may not have freeable memory in nodemask */
 308        if (!is_memcg_oom(oc) && !oom_cpuset_eligible(task, oc))
 309                goto next;
 310
 311        /*
 312         * This task already has access to memory reserves and is being killed.
 313         * Don't allow any other task to have access to the reserves unless
 314         * the task has MMF_OOM_SKIP because chances that it would release
 315         * any memory is quite low.
 316         */
 317        if (!is_sysrq_oom(oc) && tsk_is_oom_victim(task)) {
 318                if (test_bit(MMF_OOM_SKIP, &task->signal->oom_mm->flags))
 319                        goto next;
 320                goto abort;
 321        }
 322
 323        /*
 324         * If task is allocating a lot of memory and has been marked to be
 325         * killed first if it triggers an oom, then select it.
 326         */
 327        if (oom_task_origin(task)) {
 328                points = LONG_MAX;
 329                goto select;
 330        }
 331
 332        points = oom_badness(task, oc->totalpages);
 333        if (points == LONG_MIN || points < oc->chosen_points)
 334                goto next;
 335
 336select:
 337        if (oc->chosen)
 338                put_task_struct(oc->chosen);
 339        get_task_struct(task);
 340        oc->chosen = task;
 341        oc->chosen_points = points;
 342next:
 343        return 0;
 344abort:
 345        if (oc->chosen)
 346                put_task_struct(oc->chosen);
 347        oc->chosen = (void *)-1UL;
 348        return 1;
 349}
 350
 351/*
 352 * Simple selection loop. We choose the process with the highest number of
 353 * 'points'. In case scan was aborted, oc->chosen is set to -1.
 354 */
 355static void select_bad_process(struct oom_control *oc)
 356{
 357        oc->chosen_points = LONG_MIN;
 358
 359        if (is_memcg_oom(oc))
 360                mem_cgroup_scan_tasks(oc->memcg, oom_evaluate_task, oc);
 361        else {
 362                struct task_struct *p;
 363
 364                rcu_read_lock();
 365                for_each_process(p)
 366                        if (oom_evaluate_task(p, oc))
 367                                break;
 368                rcu_read_unlock();
 369        }
 370}
 371
 372static int dump_task(struct task_struct *p, void *arg)
 373{
 374        struct oom_control *oc = arg;
 375        struct task_struct *task;
 376
 377        if (oom_unkillable_task(p))
 378                return 0;
 379
 380        /* p may not have freeable memory in nodemask */
 381        if (!is_memcg_oom(oc) && !oom_cpuset_eligible(p, oc))
 382                return 0;
 383
 384        task = find_lock_task_mm(p);
 385        if (!task) {
 386                /*
 387                 * This is a kthread or all of p's threads have already
 388                 * detached their mm's.  There's no need to report
 389                 * them; they can't be oom killed anyway.
 390                 */
 391                return 0;
 392        }
 393
 394        pr_info("[%7d] %5d %5d %8lu %8lu %8ld %8lu         %5hd %s\n",
 395                task->pid, from_kuid(&init_user_ns, task_uid(task)),
 396                task->tgid, task->mm->total_vm, get_mm_rss(task->mm),
 397                mm_pgtables_bytes(task->mm),
 398                get_mm_counter(task->mm, MM_SWAPENTS),
 399                task->signal->oom_score_adj, task->comm);
 400        task_unlock(task);
 401
 402        return 0;
 403}
 404
 405/**
 406 * dump_tasks - dump current memory state of all system tasks
 407 * @oc: pointer to struct oom_control
 408 *
 409 * Dumps the current memory state of all eligible tasks.  Tasks not in the same
 410 * memcg, not in the same cpuset, or bound to a disjoint set of mempolicy nodes
 411 * are not shown.
 412 * State information includes task's pid, uid, tgid, vm size, rss,
 413 * pgtables_bytes, swapents, oom_score_adj value, and name.
 414 */
 415static void dump_tasks(struct oom_control *oc)
 416{
 417        pr_info("Tasks state (memory values in pages):\n");
 418        pr_info("[  pid  ]   uid  tgid total_vm      rss pgtables_bytes swapents oom_score_adj name\n");
 419
 420        if (is_memcg_oom(oc))
 421                mem_cgroup_scan_tasks(oc->memcg, dump_task, oc);
 422        else {
 423                struct task_struct *p;
 424
 425                rcu_read_lock();
 426                for_each_process(p)
 427                        dump_task(p, oc);
 428                rcu_read_unlock();
 429        }
 430}
 431
 432static void dump_oom_summary(struct oom_control *oc, struct task_struct *victim)
 433{
 434        /* one line summary of the oom killer context. */
 435        pr_info("oom-kill:constraint=%s,nodemask=%*pbl",
 436                        oom_constraint_text[oc->constraint],
 437                        nodemask_pr_args(oc->nodemask));
 438        cpuset_print_current_mems_allowed();
 439        mem_cgroup_print_oom_context(oc->memcg, victim);
 440        pr_cont(",task=%s,pid=%d,uid=%d\n", victim->comm, victim->pid,
 441                from_kuid(&init_user_ns, task_uid(victim)));
 442}
 443
 444static void dump_header(struct oom_control *oc, struct task_struct *p)
 445{
 446        pr_warn("%s invoked oom-killer: gfp_mask=%#x(%pGg), order=%d, oom_score_adj=%hd\n",
 447                current->comm, oc->gfp_mask, &oc->gfp_mask, oc->order,
 448                        current->signal->oom_score_adj);
 449        if (!IS_ENABLED(CONFIG_COMPACTION) && oc->order)
 450                pr_warn("COMPACTION is disabled!!!\n");
 451
 452        dump_stack();
 453        if (is_memcg_oom(oc))
 454                mem_cgroup_print_oom_meminfo(oc->memcg);
 455        else {
 456                show_mem(SHOW_MEM_FILTER_NODES, oc->nodemask);
 457                if (is_dump_unreclaim_slabs())
 458                        dump_unreclaimable_slab();
 459        }
 460        if (sysctl_oom_dump_tasks)
 461                dump_tasks(oc);
 462        if (p)
 463                dump_oom_summary(oc, p);
 464}
 465
 466/*
 467 * Number of OOM victims in flight
 468 */
 469static atomic_t oom_victims = ATOMIC_INIT(0);
 470static DECLARE_WAIT_QUEUE_HEAD(oom_victims_wait);
 471
 472static bool oom_killer_disabled __read_mostly;
 473
 474#define K(x) ((x) << (PAGE_SHIFT-10))
 475
 476/*
 477 * task->mm can be NULL if the task is the exited group leader.  So to
 478 * determine whether the task is using a particular mm, we examine all the
 479 * task's threads: if one of those is using this mm then this task was also
 480 * using it.
 481 */
 482bool process_shares_mm(struct task_struct *p, struct mm_struct *mm)
 483{
 484        struct task_struct *t;
 485
 486        for_each_thread(p, t) {
 487                struct mm_struct *t_mm = READ_ONCE(t->mm);
 488                if (t_mm)
 489                        return t_mm == mm;
 490        }
 491        return false;
 492}
 493
 494#ifdef CONFIG_MMU
 495/*
 496 * OOM Reaper kernel thread which tries to reap the memory used by the OOM
 497 * victim (if that is possible) to help the OOM killer to move on.
 498 */
 499static struct task_struct *oom_reaper_th;
 500static DECLARE_WAIT_QUEUE_HEAD(oom_reaper_wait);
 501static struct task_struct *oom_reaper_list;
 502static DEFINE_SPINLOCK(oom_reaper_lock);
 503
 504bool __oom_reap_task_mm(struct mm_struct *mm)
 505{
 506        struct vm_area_struct *vma;
 507        bool ret = true;
 508
 509        /*
 510         * Tell all users of get_user/copy_from_user etc... that the content
 511         * is no longer stable. No barriers really needed because unmapping
 512         * should imply barriers already and the reader would hit a page fault
 513         * if it stumbled over a reaped memory.
 514         */
 515        set_bit(MMF_UNSTABLE, &mm->flags);
 516
 517        for (vma = mm->mmap ; vma; vma = vma->vm_next) {
 518                if (!can_madv_lru_vma(vma))
 519                        continue;
 520
 521                /*
 522                 * Only anonymous pages have a good chance to be dropped
 523                 * without additional steps which we cannot afford as we
 524                 * are OOM already.
 525                 *
 526                 * We do not even care about fs backed pages because all
 527                 * which are reclaimable have already been reclaimed and
 528                 * we do not want to block exit_mmap by keeping mm ref
 529                 * count elevated without a good reason.
 530                 */
 531                if (vma_is_anonymous(vma) || !(vma->vm_flags & VM_SHARED)) {
 532                        struct mmu_notifier_range range;
 533                        struct mmu_gather tlb;
 534
 535                        mmu_notifier_range_init(&range, MMU_NOTIFY_UNMAP, 0,
 536                                                vma, mm, vma->vm_start,
 537                                                vma->vm_end);
 538                        tlb_gather_mmu(&tlb, mm, range.start, range.end);
 539                        if (mmu_notifier_invalidate_range_start_nonblock(&range)) {
 540                                ret = false;
 541                                continue;
 542                        }
 543                        unmap_page_range(&tlb, vma, range.start, range.end, NULL);
 544                        mmu_notifier_invalidate_range_end(&range);
 545                        tlb_finish_mmu(&tlb, range.start, range.end);
 546                }
 547        }
 548
 549        return ret;
 550}
 551
 552/*
 553 * Reaps the address space of the give task.
 554 *
 555 * Returns true on success and false if none or part of the address space
 556 * has been reclaimed and the caller should retry later.
 557 */
 558static bool oom_reap_task_mm(struct task_struct *tsk, struct mm_struct *mm)
 559{
 560        bool ret = true;
 561
 562        if (!mmap_read_trylock(mm)) {
 563                trace_skip_task_reaping(tsk->pid);
 564                return false;
 565        }
 566
 567        /*
 568         * MMF_OOM_SKIP is set by exit_mmap when the OOM reaper can't
 569         * work on the mm anymore. The check for MMF_OOM_SKIP must run
 570         * under mmap_lock for reading because it serializes against the
 571         * mmap_write_lock();mmap_write_unlock() cycle in exit_mmap().
 572         */
 573        if (test_bit(MMF_OOM_SKIP, &mm->flags)) {
 574                trace_skip_task_reaping(tsk->pid);
 575                goto out_unlock;
 576        }
 577
 578        trace_start_task_reaping(tsk->pid);
 579
 580        /* failed to reap part of the address space. Try again later */
 581        ret = __oom_reap_task_mm(mm);
 582        if (!ret)
 583                goto out_finish;
 584
 585        pr_info("oom_reaper: reaped process %d (%s), now anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB\n",
 586                        task_pid_nr(tsk), tsk->comm,
 587                        K(get_mm_counter(mm, MM_ANONPAGES)),
 588                        K(get_mm_counter(mm, MM_FILEPAGES)),
 589                        K(get_mm_counter(mm, MM_SHMEMPAGES)));
 590out_finish:
 591        trace_finish_task_reaping(tsk->pid);
 592out_unlock:
 593        mmap_read_unlock(mm);
 594
 595        return ret;
 596}
 597
 598#define MAX_OOM_REAP_RETRIES 10
 599static void oom_reap_task(struct task_struct *tsk)
 600{
 601        int attempts = 0;
 602        struct mm_struct *mm = tsk->signal->oom_mm;
 603
 604        /* Retry the mmap_read_trylock(mm) a few times */
 605        while (attempts++ < MAX_OOM_REAP_RETRIES && !oom_reap_task_mm(tsk, mm))
 606                schedule_timeout_idle(HZ/10);
 607
 608        if (attempts <= MAX_OOM_REAP_RETRIES ||
 609            test_bit(MMF_OOM_SKIP, &mm->flags))
 610                goto done;
 611
 612        pr_info("oom_reaper: unable to reap pid:%d (%s)\n",
 613                task_pid_nr(tsk), tsk->comm);
 614        sched_show_task(tsk);
 615        debug_show_all_locks();
 616
 617done:
 618        tsk->oom_reaper_list = NULL;
 619
 620        /*
 621         * Hide this mm from OOM killer because it has been either reaped or
 622         * somebody can't call mmap_write_unlock(mm).
 623         */
 624        set_bit(MMF_OOM_SKIP, &mm->flags);
 625
 626        /* Drop a reference taken by wake_oom_reaper */
 627        put_task_struct(tsk);
 628}
 629
 630static int oom_reaper(void *unused)
 631{
 632        set_freezable();
 633
 634        while (true) {
 635                struct task_struct *tsk = NULL;
 636
 637                wait_event_freezable(oom_reaper_wait, oom_reaper_list != NULL);
 638                spin_lock(&oom_reaper_lock);
 639                if (oom_reaper_list != NULL) {
 640                        tsk = oom_reaper_list;
 641                        oom_reaper_list = tsk->oom_reaper_list;
 642                }
 643                spin_unlock(&oom_reaper_lock);
 644
 645                if (tsk)
 646                        oom_reap_task(tsk);
 647        }
 648
 649        return 0;
 650}
 651
 652static void wake_oom_reaper(struct task_struct *tsk)
 653{
 654        /* mm is already queued? */
 655        if (test_and_set_bit(MMF_OOM_REAP_QUEUED, &tsk->signal->oom_mm->flags))
 656                return;
 657
 658        get_task_struct(tsk);
 659
 660        spin_lock(&oom_reaper_lock);
 661        tsk->oom_reaper_list = oom_reaper_list;
 662        oom_reaper_list = tsk;
 663        spin_unlock(&oom_reaper_lock);
 664        trace_wake_reaper(tsk->pid);
 665        wake_up(&oom_reaper_wait);
 666}
 667
 668static int __init oom_init(void)
 669{
 670        oom_reaper_th = kthread_run(oom_reaper, NULL, "oom_reaper");
 671        return 0;
 672}
 673subsys_initcall(oom_init)
 674#else
 675static inline void wake_oom_reaper(struct task_struct *tsk)
 676{
 677}
 678#endif /* CONFIG_MMU */
 679
 680/**
 681 * mark_oom_victim - mark the given task as OOM victim
 682 * @tsk: task to mark
 683 *
 684 * Has to be called with oom_lock held and never after
 685 * oom has been disabled already.
 686 *
 687 * tsk->mm has to be non NULL and caller has to guarantee it is stable (either
 688 * under task_lock or operate on the current).
 689 */
 690static void mark_oom_victim(struct task_struct *tsk)
 691{
 692        struct mm_struct *mm = tsk->mm;
 693
 694        WARN_ON(oom_killer_disabled);
 695        /* OOM killer might race with memcg OOM */
 696        if (test_and_set_tsk_thread_flag(tsk, TIF_MEMDIE))
 697                return;
 698
 699        /* oom_mm is bound to the signal struct life time. */
 700        if (!cmpxchg(&tsk->signal->oom_mm, NULL, mm)) {
 701                mmgrab(tsk->signal->oom_mm);
 702                set_bit(MMF_OOM_VICTIM, &mm->flags);
 703        }
 704
 705        /*
 706         * Make sure that the task is woken up from uninterruptible sleep
 707         * if it is frozen because OOM killer wouldn't be able to free
 708         * any memory and livelock. freezing_slow_path will tell the freezer
 709         * that TIF_MEMDIE tasks should be ignored.
 710         */
 711        __thaw_task(tsk);
 712        atomic_inc(&oom_victims);
 713        trace_mark_victim(tsk->pid);
 714}
 715
 716/**
 717 * exit_oom_victim - note the exit of an OOM victim
 718 */
 719void exit_oom_victim(void)
 720{
 721        clear_thread_flag(TIF_MEMDIE);
 722
 723        if (!atomic_dec_return(&oom_victims))
 724                wake_up_all(&oom_victims_wait);
 725}
 726
 727/**
 728 * oom_killer_enable - enable OOM killer
 729 */
 730void oom_killer_enable(void)
 731{
 732        oom_killer_disabled = false;
 733        pr_info("OOM killer enabled.\n");
 734}
 735
 736/**
 737 * oom_killer_disable - disable OOM killer
 738 * @timeout: maximum timeout to wait for oom victims in jiffies
 739 *
 740 * Forces all page allocations to fail rather than trigger OOM killer.
 741 * Will block and wait until all OOM victims are killed or the given
 742 * timeout expires.
 743 *
 744 * The function cannot be called when there are runnable user tasks because
 745 * the userspace would see unexpected allocation failures as a result. Any
 746 * new usage of this function should be consulted with MM people.
 747 *
 748 * Returns true if successful and false if the OOM killer cannot be
 749 * disabled.
 750 */
 751bool oom_killer_disable(signed long timeout)
 752{
 753        signed long ret;
 754
 755        /*
 756         * Make sure to not race with an ongoing OOM killer. Check that the
 757         * current is not killed (possibly due to sharing the victim's memory).
 758         */
 759        if (mutex_lock_killable(&oom_lock))
 760                return false;
 761        oom_killer_disabled = true;
 762        mutex_unlock(&oom_lock);
 763
 764        ret = wait_event_interruptible_timeout(oom_victims_wait,
 765                        !atomic_read(&oom_victims), timeout);
 766        if (ret <= 0) {
 767                oom_killer_enable();
 768                return false;
 769        }
 770        pr_info("OOM killer disabled.\n");
 771
 772        return true;
 773}
 774
 775static inline bool __task_will_free_mem(struct task_struct *task)
 776{
 777        struct signal_struct *sig = task->signal;
 778
 779        /*
 780         * A coredumping process may sleep for an extended period in exit_mm(),
 781         * so the oom killer cannot assume that the process will promptly exit
 782         * and release memory.
 783         */
 784        if (sig->flags & SIGNAL_GROUP_COREDUMP)
 785                return false;
 786
 787        if (sig->flags & SIGNAL_GROUP_EXIT)
 788                return true;
 789
 790        if (thread_group_empty(task) && (task->flags & PF_EXITING))
 791                return true;
 792
 793        return false;
 794}
 795
 796/*
 797 * Checks whether the given task is dying or exiting and likely to
 798 * release its address space. This means that all threads and processes
 799 * sharing the same mm have to be killed or exiting.
 800 * Caller has to make sure that task->mm is stable (hold task_lock or
 801 * it operates on the current).
 802 */
 803static bool task_will_free_mem(struct task_struct *task)
 804{
 805        struct mm_struct *mm = task->mm;
 806        struct task_struct *p;
 807        bool ret = true;
 808
 809        /*
 810         * Skip tasks without mm because it might have passed its exit_mm and
 811         * exit_oom_victim. oom_reaper could have rescued that but do not rely
 812         * on that for now. We can consider find_lock_task_mm in future.
 813         */
 814        if (!mm)
 815                return false;
 816
 817        if (!__task_will_free_mem(task))
 818                return false;
 819
 820        /*
 821         * This task has already been drained by the oom reaper so there are
 822         * only small chances it will free some more
 823         */
 824        if (test_bit(MMF_OOM_SKIP, &mm->flags))
 825                return false;
 826
 827        if (atomic_read(&mm->mm_users) <= 1)
 828                return true;
 829
 830        /*
 831         * Make sure that all tasks which share the mm with the given tasks
 832         * are dying as well to make sure that a) nobody pins its mm and
 833         * b) the task is also reapable by the oom reaper.
 834         */
 835        rcu_read_lock();
 836        for_each_process(p) {
 837                if (!process_shares_mm(p, mm))
 838                        continue;
 839                if (same_thread_group(task, p))
 840                        continue;
 841                ret = __task_will_free_mem(p);
 842                if (!ret)
 843                        break;
 844        }
 845        rcu_read_unlock();
 846
 847        return ret;
 848}
 849
 850static void __oom_kill_process(struct task_struct *victim, const char *message)
 851{
 852        struct task_struct *p;
 853        struct mm_struct *mm;
 854        bool can_oom_reap = true;
 855
 856        p = find_lock_task_mm(victim);
 857        if (!p) {
 858                pr_info("%s: OOM victim %d (%s) is already exiting. Skip killing the task\n",
 859                        message, task_pid_nr(victim), victim->comm);
 860                put_task_struct(victim);
 861                return;
 862        } else if (victim != p) {
 863                get_task_struct(p);
 864                put_task_struct(victim);
 865                victim = p;
 866        }
 867
 868        /* Get a reference to safely compare mm after task_unlock(victim) */
 869        mm = victim->mm;
 870        mmgrab(mm);
 871
 872        /* Raise event before sending signal: task reaper must see this */
 873        count_vm_event(OOM_KILL);
 874        memcg_memory_event_mm(mm, MEMCG_OOM_KILL);
 875
 876        /*
 877         * We should send SIGKILL before granting access to memory reserves
 878         * in order to prevent the OOM victim from depleting the memory
 879         * reserves from the user space under its control.
 880         */
 881        do_send_sig_info(SIGKILL, SEND_SIG_PRIV, victim, PIDTYPE_TGID);
 882        mark_oom_victim(victim);
 883        pr_err("%s: Killed process %d (%s) total-vm:%lukB, anon-rss:%lukB, file-rss:%lukB, shmem-rss:%lukB, UID:%u pgtables:%lukB oom_score_adj:%hd\n",
 884                message, task_pid_nr(victim), victim->comm, K(mm->total_vm),
 885                K(get_mm_counter(mm, MM_ANONPAGES)),
 886                K(get_mm_counter(mm, MM_FILEPAGES)),
 887                K(get_mm_counter(mm, MM_SHMEMPAGES)),
 888                from_kuid(&init_user_ns, task_uid(victim)),
 889                mm_pgtables_bytes(mm) >> 10, victim->signal->oom_score_adj);
 890        task_unlock(victim);
 891
 892        /*
 893         * Kill all user processes sharing victim->mm in other thread groups, if
 894         * any.  They don't get access to memory reserves, though, to avoid
 895         * depletion of all memory.  This prevents mm->mmap_lock livelock when an
 896         * oom killed thread cannot exit because it requires the semaphore and
 897         * its contended by another thread trying to allocate memory itself.
 898         * That thread will now get access to memory reserves since it has a
 899         * pending fatal signal.
 900         */
 901        rcu_read_lock();
 902        for_each_process(p) {
 903                if (!process_shares_mm(p, mm))
 904                        continue;
 905                if (same_thread_group(p, victim))
 906                        continue;
 907                if (is_global_init(p)) {
 908                        can_oom_reap = false;
 909                        set_bit(MMF_OOM_SKIP, &mm->flags);
 910                        pr_info("oom killer %d (%s) has mm pinned by %d (%s)\n",
 911                                        task_pid_nr(victim), victim->comm,
 912                                        task_pid_nr(p), p->comm);
 913                        continue;
 914                }
 915                /*
 916                 * No kthead_use_mm() user needs to read from the userspace so
 917                 * we are ok to reap it.
 918                 */
 919                if (unlikely(p->flags & PF_KTHREAD))
 920                        continue;
 921                do_send_sig_info(SIGKILL, SEND_SIG_PRIV, p, PIDTYPE_TGID);
 922        }
 923        rcu_read_unlock();
 924
 925        if (can_oom_reap)
 926                wake_oom_reaper(victim);
 927
 928        mmdrop(mm);
 929        put_task_struct(victim);
 930}
 931#undef K
 932
 933/*
 934 * Kill provided task unless it's secured by setting
 935 * oom_score_adj to OOM_SCORE_ADJ_MIN.
 936 */
 937static int oom_kill_memcg_member(struct task_struct *task, void *message)
 938{
 939        if (task->signal->oom_score_adj != OOM_SCORE_ADJ_MIN &&
 940            !is_global_init(task)) {
 941                get_task_struct(task);
 942                __oom_kill_process(task, message);
 943        }
 944        return 0;
 945}
 946
 947static void oom_kill_process(struct oom_control *oc, const char *message)
 948{
 949        struct task_struct *victim = oc->chosen;
 950        struct mem_cgroup *oom_group;
 951        static DEFINE_RATELIMIT_STATE(oom_rs, DEFAULT_RATELIMIT_INTERVAL,
 952                                              DEFAULT_RATELIMIT_BURST);
 953
 954        /*
 955         * If the task is already exiting, don't alarm the sysadmin or kill
 956         * its children or threads, just give it access to memory reserves
 957         * so it can die quickly
 958         */
 959        task_lock(victim);
 960        if (task_will_free_mem(victim)) {
 961                mark_oom_victim(victim);
 962                wake_oom_reaper(victim);
 963                task_unlock(victim);
 964                put_task_struct(victim);
 965                return;
 966        }
 967        task_unlock(victim);
 968
 969        if (__ratelimit(&oom_rs))
 970                dump_header(oc, victim);
 971
 972        /*
 973         * Do we need to kill the entire memory cgroup?
 974         * Or even one of the ancestor memory cgroups?
 975         * Check this out before killing the victim task.
 976         */
 977        oom_group = mem_cgroup_get_oom_group(victim, oc->memcg);
 978
 979        __oom_kill_process(victim, message);
 980
 981        /*
 982         * If necessary, kill all tasks in the selected memory cgroup.
 983         */
 984        if (oom_group) {
 985                mem_cgroup_print_oom_group(oom_group);
 986                mem_cgroup_scan_tasks(oom_group, oom_kill_memcg_member,
 987                                      (void*)message);
 988                mem_cgroup_put(oom_group);
 989        }
 990}
 991
 992/*
 993 * Determines whether the kernel must panic because of the panic_on_oom sysctl.
 994 */
 995static void check_panic_on_oom(struct oom_control *oc)
 996{
 997        if (likely(!sysctl_panic_on_oom))
 998                return;
 999        if (sysctl_panic_on_oom != 2) {
1000                /*
1001                 * panic_on_oom == 1 only affects CONSTRAINT_NONE, the kernel
1002                 * does not panic for cpuset, mempolicy, or memcg allocation
1003                 * failures.
1004                 */
1005                if (oc->constraint != CONSTRAINT_NONE)
1006                        return;
1007        }
1008        /* Do not panic for oom kills triggered by sysrq */
1009        if (is_sysrq_oom(oc))
1010                return;
1011        dump_header(oc, NULL);
1012        panic("Out of memory: %s panic_on_oom is enabled\n",
1013                sysctl_panic_on_oom == 2 ? "compulsory" : "system-wide");
1014}
1015
1016static BLOCKING_NOTIFIER_HEAD(oom_notify_list);
1017
1018int register_oom_notifier(struct notifier_block *nb)
1019{
1020        return blocking_notifier_chain_register(&oom_notify_list, nb);
1021}
1022EXPORT_SYMBOL_GPL(register_oom_notifier);
1023
1024int unregister_oom_notifier(struct notifier_block *nb)
1025{
1026        return blocking_notifier_chain_unregister(&oom_notify_list, nb);
1027}
1028EXPORT_SYMBOL_GPL(unregister_oom_notifier);
1029
1030/**
1031 * out_of_memory - kill the "best" process when we run out of memory
1032 * @oc: pointer to struct oom_control
1033 *
1034 * If we run out of memory, we have the choice between either
1035 * killing a random task (bad), letting the system crash (worse)
1036 * OR try to be smart about which process to kill. Note that we
1037 * don't have to be perfect here, we just have to be good.
1038 */
1039bool out_of_memory(struct oom_control *oc)
1040{
1041        unsigned long freed = 0;
1042
1043        if (oom_killer_disabled)
1044                return false;
1045
1046        if (!is_memcg_oom(oc)) {
1047                blocking_notifier_call_chain(&oom_notify_list, 0, &freed);
1048                if (freed > 0)
1049                        /* Got some memory back in the last second. */
1050                        return true;
1051        }
1052
1053        /*
1054         * If current has a pending SIGKILL or is exiting, then automatically
1055         * select it.  The goal is to allow it to allocate so that it may
1056         * quickly exit and free its memory.
1057         */
1058        if (task_will_free_mem(current)) {
1059                mark_oom_victim(current);
1060                wake_oom_reaper(current);
1061                return true;
1062        }
1063
1064        /*
1065         * The OOM killer does not compensate for IO-less reclaim.
1066         * pagefault_out_of_memory lost its gfp context so we have to
1067         * make sure exclude 0 mask - all other users should have at least
1068         * ___GFP_DIRECT_RECLAIM to get here. But mem_cgroup_oom() has to
1069         * invoke the OOM killer even if it is a GFP_NOFS allocation.
1070         */
1071        if (oc->gfp_mask && !(oc->gfp_mask & __GFP_FS) && !is_memcg_oom(oc))
1072                return true;
1073
1074        /*
1075         * Check if there were limitations on the allocation (only relevant for
1076         * NUMA and memcg) that may require different handling.
1077         */
1078        oc->constraint = constrained_alloc(oc);
1079        if (oc->constraint != CONSTRAINT_MEMORY_POLICY)
1080                oc->nodemask = NULL;
1081        check_panic_on_oom(oc);
1082
1083        if (!is_memcg_oom(oc) && sysctl_oom_kill_allocating_task &&
1084            current->mm && !oom_unkillable_task(current) &&
1085            oom_cpuset_eligible(current, oc) &&
1086            current->signal->oom_score_adj != OOM_SCORE_ADJ_MIN) {
1087                get_task_struct(current);
1088                oc->chosen = current;
1089                oom_kill_process(oc, "Out of memory (oom_kill_allocating_task)");
1090                return true;
1091        }
1092
1093        select_bad_process(oc);
1094        /* Found nothing?!?! */
1095        if (!oc->chosen) {
1096                dump_header(oc, NULL);
1097                pr_warn("Out of memory and no killable processes...\n");
1098                /*
1099                 * If we got here due to an actual allocation at the
1100                 * system level, we cannot survive this and will enter
1101                 * an endless loop in the allocator. Bail out now.
1102                 */
1103                if (!is_sysrq_oom(oc) && !is_memcg_oom(oc))
1104                        panic("System is deadlocked on memory\n");
1105        }
1106        if (oc->chosen && oc->chosen != (void *)-1UL) {
1107                oom_kill_process(oc, !is_memcg_oom(oc) ? "Out of memory" :
1108                                 "Memory cgroup out of memory");
1109                /*
1110                 * Give the killed process a good chance to exit before trying
1111                 * to allocate memory again.
1112                 */
1113                schedule_timeout_killable(1);
1114        }
1115        return !!oc->chosen;
1116}
1117
1118/*
1119 * The pagefault handler calls here because it is out of memory, so kill a
1120 * memory-hogging task. If oom_lock is held by somebody else, a parallel oom
1121 * killing is already in progress so do nothing.
1122 */
1123void pagefault_out_of_memory(void)
1124{
1125        struct oom_control oc = {
1126                .zonelist = NULL,
1127                .nodemask = NULL,
1128                .memcg = NULL,
1129                .gfp_mask = 0,
1130                .order = 0,
1131        };
1132
1133        if (mem_cgroup_oom_synchronize(true))
1134                return;
1135
1136        if (!mutex_trylock(&oom_lock))
1137                return;
1138        out_of_memory(&oc);
1139        mutex_unlock(&oom_lock);
1140}
1141