AcoSail/kernel_optimize_test
8
0
Fork 0
forked from luck/tmp_suning_uos_patched
Code Pull Requests Activity

mm: remove quicklist page table caches

Browse Source 
Patch series "mm: remove quicklist page table caches".

A while ago Nicholas proposed to remove quicklist page table caches [1].

I've rebased his patch on the curren upstream and switched ia64 and sh to
use generic versions of PTE allocation.

[1] https://lore.kernel.org/linux-mm/20190711030339.20892-1-npiggin@gmail.com

This patch (of 3):

Remove page table allocator "quicklists".  These have been around for a
long time, but have not got much traction in the last decade and are only
used on ia64 and sh architectures.

The numbers in the initial commit look interesting but probably don't
apply anymore.  If anybody wants to resurrect this it's in the git
history, but it's unhelpful to have this code and divergent allocator
behaviour for minor archs.

Also it might be better to instead make more general improvements to page
allocator if this is still so slow.

Link: http://lkml.kernel.org/r/1565250728-21721-2-git-send-email-rppt@linux.ibm.com
Signed-off-by: Nicholas Piggin <npiggin@gmail.com>
Signed-off-by: Mike Rapoport <rppt@linux.ibm.com>
Cc: Tony Luck <tony.luck@intel.com>
Cc: Yoshinori Sato <ysato@users.sourceforge.jp>
Signed-off-by: Andrew Morton <akpm@linux-foundation.org>
Signed-off-by: Linus Torvalds <torvalds@linux-foundation.org>
This commit is contained in:
Nicholas Piggin 2019-09-23 15:35:19 -07:00 committed by Linus Torvalds
parent 7b167b6810
commit 13224794cb
39 changed files with 25 additions and 395 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
arch/alpha/include/asm/pgalloc.h
View File

@ -53,6 +53,4 @@ pmd_free(struct mm_struct *mm, pmd_t *pmd)
free_page((unsigned long)pmd); free_page((unsigned long)pmd);
} }
#define check_pgt_cache() do { } while (0)
#endif /* _ALPHA_PGALLOC_H */ #endif /* _ALPHA_PGALLOC_H */

1
arch/arc/include/asm/pgalloc.h
View File

@ -129,7 +129,6 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptep)
#define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte) #define __pte_free_tlb(tlb, pte, addr) pte_free((tlb)->mm, pte)
#define check_pgt_cache() do { } while (0)
#define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd)) #define pmd_pgtable(pmd) ((pgtable_t) pmd_page_vaddr(pmd))
#endif /* _ASM_ARC_PGALLOC_H */ #endif /* _ASM_ARC_PGALLOC_H */

2
arch/arm/include/asm/pgalloc.h
View File

@ -15,8 +15,6 @@
#include <asm/cacheflush.h> #include <asm/cacheflush.h>
#include <asm/tlbflush.h> #include <asm/tlbflush.h>
#define check_pgt_cache() do { } while (0)
#ifdef CONFIG_MMU #ifdef CONFIG_MMU
#define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER)) #define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_BIT4 | PMD_DOMAIN(DOMAIN_USER))

2
arch/arm64/include/asm/pgalloc.h
View File

@ -15,8 +15,6 @@
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ #include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
#define check_pgt_cache() do { } while (0)
#define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t)) #define PGD_SIZE (PTRS_PER_PGD * sizeof(pgd_t))
#if CONFIG_PGTABLE_LEVELS > 2 #if CONFIG_PGTABLE_LEVELS > 2

2
arch/csky/include/asm/pgalloc.h
View File

@ -75,8 +75,6 @@ do { \
tlb_remove_page(tlb, pte); \ tlb_remove_page(tlb, pte); \
} while (0) } while (0)
#define check_pgt_cache() do {} while (0)
extern void pagetable_init(void); extern void pagetable_init(void);
extern void pre_mmu_init(void); extern void pre_mmu_init(void);
extern void pre_trap_init(void); extern void pre_trap_init(void);

2
arch/hexagon/include/asm/pgalloc.h
View File

@ -13,8 +13,6 @@
#include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */ #include <asm-generic/pgalloc.h> /* for pte_{alloc,free}_one */
#define check_pgt_cache() do {} while (0)
extern unsigned long long kmap_generation; extern unsigned long long kmap_generation;
/* /*

4
arch/ia64/Kconfig
View File

@ -72,10 +72,6 @@ config 64BIT
config ZONE_DMA32 config ZONE_DMA32
def_bool y def_bool y
config QUICKLIST
bool
default y
config MMU config MMU
bool bool
default y default y

32
arch/ia64/include/asm/pgalloc.h
View File

@ -19,18 +19,17 @@
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/page-flags.h> #include <linux/page-flags.h>
#include <linux/threads.h> #include <linux/threads.h>
#include <linux/quicklist.h>
#include <asm/mmu_context.h> #include <asm/mmu_context.h>
static inline pgd_t *pgd_alloc(struct mm_struct *mm) static inline pgd_t *pgd_alloc(struct mm_struct *mm)
{ {
return quicklist_alloc(0, GFP_KERNEL, NULL); return (pgd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
} }
static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd) static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
{ {
quicklist_free(0, NULL, pgd); free_page((unsigned long)pgd);
} }
#if CONFIG_PGTABLE_LEVELS == 4 #if CONFIG_PGTABLE_LEVELS == 4
@ -42,12 +41,12 @@ pgd_populate(struct mm_struct *mm, pgd_t * pgd_entry, pud_t * pud)
static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr)
{ {
return quicklist_alloc(0, GFP_KERNEL, NULL); return (pud_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
} }
static inline void pud_free(struct mm_struct *mm, pud_t *pud) static inline void pud_free(struct mm_struct *mm, pud_t *pud)
{ {
quicklist_free(0, NULL, pud); free_page((unsigned long)pud);
} }
#define __pud_free_tlb(tlb, pud, address) pud_free((tlb)->mm, pud) #define __pud_free_tlb(tlb, pud, address) pud_free((tlb)->mm, pud)
#endif /* CONFIG_PGTABLE_LEVELS == 4 */ #endif /* CONFIG_PGTABLE_LEVELS == 4 */
@ -60,12 +59,12 @@ pud_populate(struct mm_struct *mm, pud_t * pud_entry, pmd_t * pmd)
static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr)
{ {
return quicklist_alloc(0, GFP_KERNEL, NULL); return (pmd_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
} }
static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
{ {
quicklist_free(0, NULL, pmd); free_page((unsigned long)pmd);
} }
#define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd) #define __pmd_free_tlb(tlb, pmd, address) pmd_free((tlb)->mm, pmd)
@ -86,14 +85,12 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t * pmd_entry, pte_t * pte)
static inline pgtable_t pte_alloc_one(struct mm_struct *mm) static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
{ {
struct page *page; struct page *page;
void *pg;
pg = quicklist_alloc(0, GFP_KERNEL, NULL); page = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!pg) if (!page)
return NULL; return NULL;
page = virt_to_page(pg);
if (!pgtable_page_ctor(page)) { if (!pgtable_page_ctor(page)) {
quicklist_free(0, NULL, pg); __free_page(page);
return NULL; return NULL;
} }
return page; return page;
@ -101,23 +98,18 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
{ {
return quicklist_alloc(0, GFP_KERNEL, NULL); return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
} }
static inline void pte_free(struct mm_struct *mm, pgtable_t pte) static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{ {
pgtable_page_dtor(pte); pgtable_page_dtor(pte);
quicklist_free_page(0, NULL, pte); __free_page(pte);
} }
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{ {
quicklist_free(0, NULL, pte); free_page((unsigned long)pte);
}
static inline void check_pgt_cache(void)
{
quicklist_trim(0, NULL, 25, 16);
} }
#define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte) #define __pte_free_tlb(tlb, pte, address) pte_free((tlb)->mm, pte)

2
arch/m68k/include/asm/pgtable_mm.h
View File

@ -181,6 +181,4 @@ pgprot_t pgprot_dmacoherent(pgprot_t prot);
*/ */
#define pgtable_cache_init() do { } while (0) #define pgtable_cache_init() do { } while (0)
#define check_pgt_cache() do { } while (0)
#endif /* _M68K_PGTABLE_H */ #endif /* _M68K_PGTABLE_H */

2
arch/m68k/include/asm/pgtable_no.h
View File

@ -60,6 +60,4 @@ extern void paging_init(void);
#include <asm-generic/pgtable.h> #include <asm-generic/pgtable.h>
#define check_pgt_cache() do { } while (0)
#endif /* _M68KNOMMU_PGTABLE_H */ #endif /* _M68KNOMMU_PGTABLE_H */

89
arch/microblaze/include/asm/pgalloc.h
View File

@ -21,83 +21,20 @@
#include <asm/cache.h> #include <asm/cache.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#define PGDIR_ORDER 0
/*
* This is handled very differently on MicroBlaze since out page tables
* are all 0's and I want to be able to use these zero'd pages elsewhere
* as well - it gives us quite a speedup.
* -- Cort
*/
extern struct pgtable_cache_struct {
unsigned long *pgd_cache;
unsigned long *pte_cache;
unsigned long pgtable_cache_sz;
} quicklists;
#define pgd_quicklist (quicklists.pgd_cache)
#define pmd_quicklist ((unsigned long *)0)
#define pte_quicklist (quicklists.pte_cache)
#define pgtable_cache_size (quicklists.pgtable_cache_sz)
extern unsigned long *zero_cache; /* head linked list of pre-zero'd pages */
extern atomic_t zero_sz; /* # currently pre-zero'd pages */
extern atomic_t zeropage_hits; /* # zero'd pages request that we've done */
extern atomic_t zeropage_calls; /* # zero'd pages request that've been made */
extern atomic_t zerototal; /* # pages zero'd over time */
#define zero_quicklist (zero_cache)
#define zero_cache_sz (zero_sz)
#define zero_cache_calls (zeropage_calls)
#define zero_cache_hits (zeropage_hits)
#define zero_cache_total (zerototal)
/*
* return a pre-zero'd page from the list,
* return NULL if none available -- Cort
*/
extern unsigned long get_zero_page_fast(void);
extern void __bad_pte(pmd_t *pmd); extern void __bad_pte(pmd_t *pmd);
static inline pgd_t *get_pgd_slow(void) static inline pgd_t *get_pgd(void)
{ {
pgd_t *ret; return (pgd_t *)__get_free_pages(GFP_KERNEL|__GFP_ZERO, 0);
ret = (pgd_t *)__get_free_pages(GFP_KERNEL, PGDIR_ORDER);
if (ret != NULL)
clear_page(ret);
return ret;
} }
static inline pgd_t *get_pgd_fast(void) static inline void free_pgd(pgd_t *pgd)
{
unsigned long *ret;
ret = pgd_quicklist;
if (ret != NULL) {
pgd_quicklist = (unsigned long *)(*ret);
ret[0] = 0;
pgtable_cache_size--;
} else
ret = (unsigned long *)get_pgd_slow();
return (pgd_t *)ret;
}
static inline void free_pgd_fast(pgd_t *pgd)
{
*(unsigned long **)pgd = pgd_quicklist;
pgd_quicklist = (unsigned long *) pgd;
pgtable_cache_size++;
}
static inline void free_pgd_slow(pgd_t *pgd)
{ {
free_page((unsigned long)pgd); free_page((unsigned long)pgd);
} }
#define pgd_free(mm, pgd) free_pgd_fast(pgd) #define pgd_free(mm, pgd) free_pgd(pgd)
#define pgd_alloc(mm) get_pgd_fast() #define pgd_alloc(mm) get_pgd()
#define pmd_pgtable(pmd) pmd_page(pmd) #define pmd_pgtable(pmd) pmd_page(pmd)
@ -115,15 +52,14 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm)
struct page *ptepage; struct page *ptepage;
#ifdef CONFIG_HIGHPTE #ifdef CONFIG_HIGHPTE
int flags = GFP_KERNEL | __GFP_HIGHMEM; int flags = GFP_KERNEL | __GFP_ZERO | __GFP_HIGHMEM;
#else #else
int flags = GFP_KERNEL; int flags = GFP_KERNEL | __GFP_ZERO;
#endif #endif
ptepage = alloc_pages(flags, 0); ptepage = alloc_pages(flags, 0);
if (!ptepage) if (!ptepage)
return NULL; return NULL;
clear_highpage(ptepage);
if (!pgtable_page_ctor(ptepage)) { if (!pgtable_page_ctor(ptepage)) {
__free_page(ptepage); __free_page(ptepage);
return NULL; return NULL;
@ -131,13 +67,6 @@ static inline struct page *pte_alloc_one(struct mm_struct *mm)
return ptepage; return ptepage;
} }
static inline void pte_free_fast(pte_t *pte)
{
*(unsigned long **)pte = pte_quicklist;
pte_quicklist = (unsigned long *) pte;
pgtable_cache_size++;
}
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{ {
free_page((unsigned long)pte); free_page((unsigned long)pte);
@ -171,10 +100,6 @@ static inline void pte_free(struct mm_struct *mm, struct page *ptepage)
#define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x) #define __pmd_free_tlb(tlb, x, addr) pmd_free((tlb)->mm, x)
#define pgd_populate(mm, pmd, pte) BUG() #define pgd_populate(mm, pmd, pte) BUG()
extern int do_check_pgt_cache(int, int);
#endif /* CONFIG_MMU */ #endif /* CONFIG_MMU */
#define check_pgt_cache() do { } while (0)
#endif /* _ASM_MICROBLAZE_PGALLOC_H */ #endif /* _ASM_MICROBLAZE_PGALLOC_H */

4
arch/microblaze/mm/pgtable.c
View File

@ -44,10 +44,6 @@ unsigned long ioremap_base;
unsigned long ioremap_bot; unsigned long ioremap_bot;
EXPORT_SYMBOL(ioremap_bot); EXPORT_SYMBOL(ioremap_bot);
#ifndef CONFIG_SMP
struct pgtable_cache_struct quicklists;
#endif
static void __iomem *__ioremap(phys_addr_t addr, unsigned long size, static void __iomem *__ioremap(phys_addr_t addr, unsigned long size,
unsigned long flags) unsigned long flags)
{ {

2
arch/mips/include/asm/pgalloc.h
View File

@ -105,8 +105,6 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud)
#endif /* __PAGETABLE_PUD_FOLDED */ #endif /* __PAGETABLE_PUD_FOLDED */
#define check_pgt_cache() do { } while (0)
extern void pagetable_init(void); extern void pagetable_init(void);
#endif /* _ASM_PGALLOC_H */ #endif /* _ASM_PGALLOC_H */

2
arch/nds32/include/asm/pgalloc.h
View File

@ -23,8 +23,6 @@
extern pgd_t *pgd_alloc(struct mm_struct *mm); extern pgd_t *pgd_alloc(struct mm_struct *mm);
extern void pgd_free(struct mm_struct *mm, pgd_t * pgd); extern void pgd_free(struct mm_struct *mm, pgd_t * pgd);
#define check_pgt_cache() do { } while (0)
static inline pgtable_t pte_alloc_one(struct mm_struct *mm) static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
{ {
pgtable_t pte; pgtable_t pte;

2
arch/nios2/include/asm/pgalloc.h
View File

@ -45,6 +45,4 @@ static inline void pgd_free(struct mm_struct *mm, pgd_t *pgd)
tlb_remove_page((tlb), (pte)); \ tlb_remove_page((tlb), (pte)); \
} while (0) } while (0)
#define check_pgt_cache() do { } while (0)
#endif /* _ASM_NIOS2_PGALLOC_H */ #endif /* _ASM_NIOS2_PGALLOC_H */

2
arch/openrisc/include/asm/pgalloc.h
View File

@ -101,6 +101,4 @@ do { \
#define pmd_pgtable(pmd) pmd_page(pmd) #define pmd_pgtable(pmd) pmd_page(pmd)
#define check_pgt_cache() do { } while (0)
#endif #endif

2
arch/parisc/include/asm/pgalloc.h
View File

@ -124,6 +124,4 @@ pmd_populate_kernel(struct mm_struct *mm, pmd_t *pmd, pte_t *pte)
pmd_populate_kernel(mm, pmd, page_address(pte_page)) pmd_populate_kernel(mm, pmd, page_address(pte_page))
#define pmd_pgtable(pmd) pmd_page(pmd) #define pmd_pgtable(pmd) pmd_page(pmd)
#define check_pgt_cache() do { } while (0)
#endif #endif

2
arch/powerpc/include/asm/pgalloc.h
View File

@ -64,8 +64,6 @@ static inline void pte_free(struct mm_struct *mm, pgtable_t ptepage)
extern struct kmem_cache *pgtable_cache[]; extern struct kmem_cache *pgtable_cache[];
#define PGT_CACHE(shift) pgtable_cache[shift] #define PGT_CACHE(shift) pgtable_cache[shift]
static inline void check_pgt_cache(void) { }
#ifdef CONFIG_PPC_BOOK3S #ifdef CONFIG_PPC_BOOK3S
#include <asm/book3s/pgalloc.h> #include <asm/book3s/pgalloc.h>
#else #else

4
arch/riscv/include/asm/pgalloc.h
View File

@ -82,8 +82,4 @@ do { \
tlb_remove_page((tlb), pte); \ tlb_remove_page((tlb), pte); \
} while (0) } while (0)
static inline void check_pgt_cache(void)
{
}
#endif /* _ASM_RISCV_PGALLOC_H */ #endif /* _ASM_RISCV_PGALLOC_H */

1
arch/s390/include/asm/pgtable.h
View File

@ -1686,7 +1686,6 @@ extern void s390_reset_cmma(struct mm_struct *mm);
* No page table caches to initialise * No page table caches to initialise
*/ */
static inline void pgtable_cache_init(void) { } static inline void pgtable_cache_init(void) { }
static inline void check_pgt_cache(void) { }
#include <asm-generic/pgtable.h> #include <asm-generic/pgtable.h>

22
arch/sh/include/asm/pgalloc.h
View File

@ -2,11 +2,8 @@
#ifndef __ASM_SH_PGALLOC_H #ifndef __ASM_SH_PGALLOC_H
#define __ASM_SH_PGALLOC_H #define __ASM_SH_PGALLOC_H
#include <linux/quicklist.h>
#include <asm/page.h> #include <asm/page.h>
#define QUICK_PT 0 /* Other page table pages that are zero on free */
extern pgd_t *pgd_alloc(struct mm_struct *); extern pgd_t *pgd_alloc(struct mm_struct *);
extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd);
@ -34,20 +31,18 @@ static inline void pmd_populate(struct mm_struct *mm, pmd_t *pmd,
*/ */
static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm) static inline pte_t *pte_alloc_one_kernel(struct mm_struct *mm)
{ {
return quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); return (pte_t *)__get_free_page(GFP_KERNEL | __GFP_ZERO);
} }
static inline pgtable_t pte_alloc_one(struct mm_struct *mm) static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
{ {
struct page *page; struct page *page;
void *pg;
pg = quicklist_alloc(QUICK_PT, GFP_KERNEL, NULL); page = alloc_page(GFP_KERNEL | __GFP_ZERO);
if (!pg) if (!page)
return NULL; return NULL;
page = virt_to_page(pg);
if (!pgtable_page_ctor(page)) { if (!pgtable_page_ctor(page)) {
quicklist_free(QUICK_PT, NULL, pg); __free_page(page);
return NULL; return NULL;
} }
return page; return page;
@ -55,13 +50,13 @@ static inline pgtable_t pte_alloc_one(struct mm_struct *mm)
static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte) static inline void pte_free_kernel(struct mm_struct *mm, pte_t *pte)
{ {
quicklist_free(QUICK_PT, NULL, pte); free_page((unsigned long)pte);
} }
static inline void pte_free(struct mm_struct *mm, pgtable_t pte) static inline void pte_free(struct mm_struct *mm, pgtable_t pte)
{ {
pgtable_page_dtor(pte); pgtable_page_dtor(pte);
quicklist_free_page(QUICK_PT, NULL, pte); __free_page(pte);
} }
#define __pte_free_tlb(tlb,pte,addr) \ #define __pte_free_tlb(tlb,pte,addr) \
@ -79,9 +74,4 @@ do { \
} while (0); } while (0);
#endif #endif
static inline void check_pgt_cache(void)
{
quicklist_trim(QUICK_PT, NULL, 25, 16);
}
#endif /* __ASM_SH_PGALLOC_H */ #endif /* __ASM_SH_PGALLOC_H */

3
arch/sh/mm/Kconfig
View File

@ -1,9 +1,6 @@
# SPDX-License-Identifier: GPL-2.0 # SPDX-License-Identifier: GPL-2.0
menu "Memory management options" menu "Memory management options"
config QUICKLIST
def_bool y
config MMU config MMU
bool "Support for memory management hardware" bool "Support for memory management hardware"
depends on !CPU_SH2 depends on !CPU_SH2

2
arch/sparc/include/asm/pgalloc_32.h
View File

@ -17,8 +17,6 @@ void srmmu_free_nocache(void *addr, int size);
extern struct resource sparc_iomap; extern struct resource sparc_iomap;
#define check_pgt_cache() do { } while (0)
pgd_t *get_pgd_fast(void); pgd_t *get_pgd_fast(void);
static inline void free_pgd_fast(pgd_t *pgd) static inline void free_pgd_fast(pgd_t *pgd)
{ {

2
arch/sparc/include/asm/pgalloc_64.h
View File

@ -69,8 +69,6 @@ void pte_free(struct mm_struct *mm, pgtable_t ptepage);
#define pmd_populate(MM, PMD, PTE) pmd_set(MM, PMD, PTE) #define pmd_populate(MM, PMD, PTE) pmd_set(MM, PMD, PTE)
#define pmd_pgtable(PMD) ((pte_t *)__pmd_page(PMD)) #define pmd_pgtable(PMD) ((pte_t *)__pmd_page(PMD))
#define check_pgt_cache() do { } while (0)
void pgtable_free(void *table, bool is_page); void pgtable_free(void *table, bool is_page);
#ifdef CONFIG_SMP #ifdef CONFIG_SMP

1
arch/sparc/mm/init_32.c
View File

@ -31,7 +31,6 @@
#include <asm/page.h> #include <asm/page.h>
#include <asm/pgtable.h> #include <asm/pgtable.h>
#include <asm/vaddrs.h> #include <asm/vaddrs.h>
#include <asm/pgalloc.h> /* bug in asm-generic/tlb.h: check_pgt_cache */
#include <asm/setup.h> #include <asm/setup.h>
#include <asm/tlb.h> #include <asm/tlb.h>
#include <asm/prom.h> #include <asm/prom.h>

2
arch/um/include/asm/pgalloc.h
View File

@ -43,7 +43,5 @@ static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd)
#define __pmd_free_tlb(tlb,x, address) tlb_remove_page((tlb),virt_to_page(x)) #define __pmd_free_tlb(tlb,x, address) tlb_remove_page((tlb),virt_to_page(x))
#endif #endif
#define check_pgt_cache() do { } while (0)
#endif #endif

2
arch/unicore32/include/asm/pgalloc.h
View File

@ -18,8 +18,6 @@
#define __HAVE_ARCH_PTE_ALLOC_ONE #define __HAVE_ARCH_PTE_ALLOC_ONE
#include <asm-generic/pgalloc.h> #include <asm-generic/pgalloc.h>
#define check_pgt_cache() do { } while (0)
#define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_PRESENT) #define _PAGE_USER_TABLE (PMD_TYPE_TABLE | PMD_PRESENT)
#define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_PRESENT) #define _PAGE_KERNEL_TABLE (PMD_TYPE_TABLE | PMD_PRESENT)

1
arch/x86/include/asm/pgtable_32.h
View File

@ -30,7 +30,6 @@ extern pgd_t initial_page_table[1024];
extern pmd_t initial_pg_pmd[]; extern pmd_t initial_pg_pmd[];
static inline void pgtable_cache_init(void) { } static inline void pgtable_cache_init(void) { }
static inline void check_pgt_cache(void) { }
void paging_init(void); void paging_init(void);
void sync_initial_page_table(void); void sync_initial_page_table(void);

1
arch/x86/include/asm/pgtable_64.h
View File

@ -242,7 +242,6 @@ extern void cleanup_highmap(void);
#define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN #define HAVE_ARCH_UNMAPPED_AREA_TOPDOWN
#define pgtable_cache_init() do { } while (0) #define pgtable_cache_init() do { } while (0)
#define check_pgt_cache() do { } while (0)
#define PAGE_AGP PAGE_KERNEL_NOCACHE #define PAGE_AGP PAGE_KERNEL_NOCACHE
#define HAVE_PAGE_AGP 1 #define HAVE_PAGE_AGP 1

3
arch/xtensa/include/asm/tlbflush.h
View File

@ -160,9 +160,6 @@ static inline void invalidate_dtlb_mapping (unsigned address)
invalidate_dtlb_entry(tlb_entry); invalidate_dtlb_entry(tlb_entry);
} }
#define check_pgt_cache() do { } while (0)
/* /*
* DO NOT USE THESE FUNCTIONS. These instructions aren't part of the Xtensa * DO NOT USE THESE FUNCTIONS. These instructions aren't part of the Xtensa
* ISA and exist only for test purposes.. * ISA and exist only for test purposes..

4
fs/proc/meminfo.c
View File

@ -8,7 +8,6 @@
#include <linux/mmzone.h> #include <linux/mmzone.h>
#include <linux/proc_fs.h> #include <linux/proc_fs.h>
#include <linux/percpu.h> #include <linux/percpu.h>
#include <linux/quicklist.h>
#include <linux/seq_file.h> #include <linux/seq_file.h>
#include <linux/swap.h> #include <linux/swap.h>
#include <linux/vmstat.h> #include <linux/vmstat.h>
@ -106,9 +105,6 @@ static int meminfo_proc_show(struct seq_file *m, void *v)
global_zone_page_state(NR_KERNEL_STACK_KB)); global_zone_page_state(NR_KERNEL_STACK_KB));
show_val_kb(m, "PageTables: ", show_val_kb(m, "PageTables: ",
global_zone_page_state(NR_PAGETABLE)); global_zone_page_state(NR_PAGETABLE));
#ifdef CONFIG_QUICKLIST
show_val_kb(m, "Quicklists: ", quicklist_total_size());
#endif
show_val_kb(m, "NFS_Unstable: ", show_val_kb(m, "NFS_Unstable: ",
global_node_page_state(NR_UNSTABLE_NFS)); global_node_page_state(NR_UNSTABLE_NFS));

5
include/asm-generic/pgalloc.h
View File

@ -102,11 +102,6 @@ static inline void pte_free(struct mm_struct *mm, struct page *pte_page)
__free_page(pte_page); __free_page(pte_page);
} }
#else /* CONFIG_MMU */
/* This is enough for a nommu architecture */
#define check_pgt_cache() do { } while (0)
#endif /* CONFIG_MMU */ #endif /* CONFIG_MMU */
#endif /* __ASM_GENERIC_PGALLOC_H */ #endif /* __ASM_GENERIC_PGALLOC_H */

94
include/linux/quicklist.h
View File

@ -1,94 +0,0 @@
/* SPDX-License-Identifier: GPL-2.0 */
#ifndef LINUX_QUICKLIST_H
#define LINUX_QUICKLIST_H
/*
* Fast allocations and disposal of pages. Pages must be in the condition
* as needed after allocation when they are freed. Per cpu lists of pages
* are kept that only contain node local pages.
*
* (C) 2007, SGI. Christoph Lameter <cl@linux.com>
*/
#include <linux/kernel.h>
#include <linux/gfp.h>
#include <linux/percpu.h>
#ifdef CONFIG_QUICKLIST
struct quicklist {
void *page;
int nr_pages;
};
DECLARE_PER_CPU(struct quicklist, quicklist)[CONFIG_NR_QUICK];
/*
* The two key functions quicklist_alloc and quicklist_free are inline so
* that they may be custom compiled for the platform.
* Specifying a NULL ctor can remove constructor support. Specifying
* a constant quicklist allows the determination of the exact address
* in the per cpu area.
*
* The fast patch in quicklist_alloc touched only a per cpu cacheline and
* the first cacheline of the page itself. There is minmal overhead involved.
*/
static inline void *quicklist_alloc(int nr, gfp_t flags, void (*ctor)(void *))
{
struct quicklist *q;
void **p = NULL;
q =&get_cpu_var(quicklist)[nr];
p = q->page;
if (likely(p)) {
q->page = p[0];
p[0] = NULL;
q->nr_pages--;
}
put_cpu_var(quicklist);
if (likely(p))
return p;
p = (void *)__get_free_page(flags | __GFP_ZERO);
if (ctor && p)
ctor(p);
return p;
}
static inline void __quicklist_free(int nr, void (*dtor)(void *), void *p,
struct page *page)
{
struct quicklist *q;
q = &get_cpu_var(quicklist)[nr];
*(void **)p = q->page;
q->page = p;
q->nr_pages++;
put_cpu_var(quicklist);
}
static inline void quicklist_free(int nr, void (*dtor)(void *), void *pp)
{
__quicklist_free(nr, dtor, pp, virt_to_page(pp));
}
static inline void quicklist_free_page(int nr, void (*dtor)(void *),
struct page *page)
{
__quicklist_free(nr, dtor, page_address(page), page);
}
void quicklist_trim(int nr, void (*dtor)(void *),
unsigned long min_pages, unsigned long max_free);
unsigned long quicklist_total_size(void);
#else
static inline unsigned long quicklist_total_size(void)
{
return 0;
}
#endif
#endif /* LINUX_QUICKLIST_H */

1
kernel/sched/idle.c
View File

@ -238,7 +238,6 @@ static void do_idle(void)
tick_nohz_idle_enter(); tick_nohz_idle_enter();
while (!need_resched()) { while (!need_resched()) {
check_pgt_cache();
rmb(); rmb();
local_irq_disable(); local_irq_disable();

5
lib/show_mem.c
View File

@ -6,7 +6,6 @@
*/ */
#include <linux/mm.h> #include <linux/mm.h>
#include <linux/quicklist.h>
#include <linux/cma.h> #include <linux/cma.h>
void show_mem(unsigned int filter, nodemask_t *nodemask) void show_mem(unsigned int filter, nodemask_t *nodemask)
@ -39,10 +38,6 @@ void show_mem(unsigned int filter, nodemask_t *nodemask)
#ifdef CONFIG_CMA #ifdef CONFIG_CMA
printk("%lu pages cma reserved\n", totalcma_pages); printk("%lu pages cma reserved\n", totalcma_pages);
#endif #endif
#ifdef CONFIG_QUICKLIST
printk("%lu pages in pagetable cache\n",
quicklist_total_size());
#endif
#ifdef CONFIG_MEMORY_FAILURE #ifdef CONFIG_MEMORY_FAILURE
printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages)); printk("%lu pages hwpoisoned\n", atomic_long_read(&num_poisoned_pages));
#endif #endif

5
mm/Kconfig
View File

@ -273,11 +273,6 @@ config BOUNCE
by default when ZONE_DMA or HIGHMEM is selected, but you by default when ZONE_DMA or HIGHMEM is selected, but you
may say n to override this. may say n to override this.
config NR_QUICK
int
depends on QUICKLIST
default "1"
config VIRT_TO_BUS config VIRT_TO_BUS
bool bool
help help

1
mm/Makefile
View File

@ -72,7 +72,6 @@ obj-$(CONFIG_FAILSLAB) += failslab.o
obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o obj-$(CONFIG_MEMORY_HOTPLUG) += memory_hotplug.o
obj-$(CONFIG_MEMTEST) += memtest.o obj-$(CONFIG_MEMTEST) += memtest.o
obj-$(CONFIG_MIGRATION) += migrate.o obj-$(CONFIG_MIGRATION) += migrate.o
obj-$(CONFIG_QUICKLIST) += quicklist.o
obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o obj-$(CONFIG_TRANSPARENT_HUGEPAGE) += huge_memory.o khugepaged.o
obj-$(CONFIG_PAGE_COUNTER) += page_counter.o obj-$(CONFIG_PAGE_COUNTER) += page_counter.o
obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o obj-$(CONFIG_MEMCG) += memcontrol.o vmpressure.o

2
mm/mmu_gather.c
View File

@ -271,8 +271,6 @@ void tlb_finish_mmu(struct mmu_gather *tlb,
tlb_flush_mmu(tlb); tlb_flush_mmu(tlb);
/* keep the page table cache within bounds */
check_pgt_cache();
#ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER #ifndef CONFIG_HAVE_MMU_GATHER_NO_GATHER
tlb_batch_list_free(tlb); tlb_batch_list_free(tlb);
#endif #endif

103
mm/quicklist.c
View File

@ -1,103 +0,0 @@
// SPDX-License-Identifier: GPL-2.0
/*
* Quicklist support.
*
* Quicklists are light weight lists of pages that have a defined state
* on alloc and free. Pages must be in the quicklist specific defined state
* (zero by default) when the page is freed. It seems that the initial idea
* for such lists first came from Dave Miller and then various other people
* improved on it.
*
* Copyright (C) 2007 SGI,
* Christoph Lameter <cl@linux.com>
* Generalized, added support for multiple lists and
* constructors / destructors.
*/
#include <linux/kernel.h>
#include <linux/gfp.h>
#include <linux/mm.h>
#include <linux/mmzone.h>
#include <linux/quicklist.h>
DEFINE_PER_CPU(struct quicklist [CONFIG_NR_QUICK], quicklist);
#define FRACTION_OF_NODE_MEM 16
static unsigned long max_pages(unsigned long min_pages)
{
unsigned long node_free_pages, max;
int node = numa_node_id();
struct zone *zones = NODE_DATA(node)->node_zones;
int num_cpus_on_node;
node_free_pages =
#ifdef CONFIG_ZONE_DMA
zone_page_state(&zones[ZONE_DMA], NR_FREE_PAGES) +
#endif
#ifdef CONFIG_ZONE_DMA32
zone_page_state(&zones[ZONE_DMA32], NR_FREE_PAGES) +
#endif
zone_page_state(&zones[ZONE_NORMAL], NR_FREE_PAGES);
max = node_free_pages / FRACTION_OF_NODE_MEM;
num_cpus_on_node = cpumask_weight(cpumask_of_node(node));
max /= num_cpus_on_node;
return max(max, min_pages);
}
static long min_pages_to_free(struct quicklist *q,
unsigned long min_pages, long max_free)
{
long pages_to_free;
pages_to_free = q->nr_pages - max_pages(min_pages);
return min(pages_to_free, max_free);
}
/*
* Trim down the number of pages in the quicklist
*/
void quicklist_trim(int nr, void (*dtor)(void *),
unsigned long min_pages, unsigned long max_free)
{
long pages_to_free;
struct quicklist *q;
q = &get_cpu_var(quicklist)[nr];
if (q->nr_pages > min_pages) {
pages_to_free = min_pages_to_free(q, min_pages, max_free);
while (pages_to_free > 0) {
/*
* We pass a gfp_t of 0 to quicklist_alloc here
* because we will never call into the page allocator.
*/
void *p = quicklist_alloc(nr, 0, NULL);
if (dtor)
dtor(p);
free_page((unsigned long)p);
pages_to_free--;
}
}
put_cpu_var(quicklist);
}
unsigned long quicklist_total_size(void)
{
unsigned long count = 0;
int cpu;
struct quicklist *ql, *q;
for_each_online_cpu(cpu) {
ql = per_cpu(quicklist, cpu);
for (q = ql; q < ql + CONFIG_NR_QUICK; q++)
count += q->nr_pages;
}
return count;
}