diff options
| author | Vlastimil Babka <vbabka@suse.cz> | 2021-05-10 13:56:17 +0200 |
|---|---|---|
| committer | Vlastimil Babka <vbabka@suse.cz> | 2021-09-04 01:12:21 +0200 |
| commit | fa417ab7506f9234100e249938d13e94a8c404e8 (patch) | |
| tree | 55012f99def7d94fcec8093dd89bd2481d5b7565 /mm/slub.c | |
| parent | 0b303fb402862dcb7948eeeed2439bd8c99948b5 (diff) | |
mm, slub: move disabling irqs closer to get_partial() in ___slab_alloc()
Continue reducing the irq disabled scope. Check for per-cpu partial slabs with
first with irqs enabled and then recheck with irqs disabled before grabbing
the slab page. Mostly preparatory for the following patches.
Signed-off-by: Vlastimil Babka <vbabka@suse.cz>
Diffstat (limited to 'mm/slub.c')
| -rw-r--r-- | mm/slub.c | 34 |
1 files changed, 25 insertions, 9 deletions
diff --git a/mm/slub.c b/mm/slub.c index 6295695d8515..4d1f3e4a5951 100644 --- a/mm/slub.c +++ b/mm/slub.c @@ -2706,11 +2706,6 @@ reread_page: if (unlikely(node != NUMA_NO_NODE && !node_isset(node, slab_nodes))) node = NUMA_NO_NODE; - local_irq_save(flags); - if (unlikely(c->page)) { - local_irq_restore(flags); - goto reread_page; - } goto new_slab; } redo: @@ -2751,6 +2746,7 @@ redo: if (!freelist) { c->page = NULL; + local_irq_restore(flags); stat(s, DEACTIVATE_BYPASS); goto new_slab; } @@ -2780,12 +2776,19 @@ deactivate_slab: goto reread_page; } deactivate_slab(s, page, c->freelist, c); + local_irq_restore(flags); new_slab: - lockdep_assert_irqs_disabled(); - if (slub_percpu_partial(c)) { + local_irq_save(flags); + if (unlikely(c->page)) { + local_irq_restore(flags); + goto reread_page; + } + if (unlikely(!slub_percpu_partial(c))) + goto new_objects; /* stolen by an IRQ handler */ + page = c->page = slub_percpu_partial(c); slub_set_percpu_partial(c, page); local_irq_restore(flags); @@ -2793,6 +2796,16 @@ new_slab: goto redo; } + local_irq_save(flags); + if (unlikely(c->page)) { + local_irq_restore(flags); + goto reread_page; + } + +new_objects: + + lockdep_assert_irqs_disabled(); + freelist = get_partial(s, gfpflags, node, &page); if (freelist) { c->page = page; @@ -2825,15 +2838,18 @@ new_slab: check_new_page: if (kmem_cache_debug(s)) { - if (!alloc_debug_processing(s, page, freelist, addr)) + if (!alloc_debug_processing(s, page, freelist, addr)) { /* Slab failed checks. Next slab needed */ + c->page = NULL; + local_irq_restore(flags); goto new_slab; - else + } else { /* * For debug case, we don't load freelist so that all * allocations go through alloc_debug_processing() */ goto return_single; + } } if (unlikely(!pfmemalloc_match(page, gfpflags))) |