diff options
Diffstat (limited to 'target/linux/generic/backport-5.15/020-v6.1-08-mm-multi-gen-LRU-support-page-table-walks.patch')
-rw-r--r-- | target/linux/generic/backport-5.15/020-v6.1-08-mm-multi-gen-LRU-support-page-table-walks.patch | 97 |
1 files changed, 37 insertions, 60 deletions
diff --git a/target/linux/generic/backport-5.15/020-v6.1-08-mm-multi-gen-LRU-support-page-table-walks.patch b/target/linux/generic/backport-5.15/020-v6.1-08-mm-multi-gen-LRU-support-page-table-walks.patch index 64de2c0c82..b0df223b7c 100644 --- a/target/linux/generic/backport-5.15/020-v6.1-08-mm-multi-gen-LRU-support-page-table-walks.patch +++ b/target/linux/generic/backport-5.15/020-v6.1-08-mm-multi-gen-LRU-support-page-table-walks.patch @@ -147,11 +147,9 @@ Signed-off-by: Andrew Morton <akpm@linux-foundation.org> mm/vmscan.c | 1010 +++++++++++++++++++++++++++++++++++- 10 files changed, 1172 insertions(+), 17 deletions(-) -diff --git a/fs/exec.c b/fs/exec.c -index 881390b44cfd..1afa15a07d26 100644 --- a/fs/exec.c +++ b/fs/exec.c -@@ -1013,6 +1013,7 @@ static int exec_mmap(struct mm_struct *mm) +@@ -1013,6 +1013,7 @@ static int exec_mmap(struct mm_struct *m active_mm = tsk->active_mm; tsk->active_mm = mm; tsk->mm = mm; @@ -159,7 +157,7 @@ index 881390b44cfd..1afa15a07d26 100644 /* * This prevents preemption while active_mm is being loaded and * it and mm are being updated, which could cause problems for -@@ -1028,6 +1029,7 @@ static int exec_mmap(struct mm_struct *mm) +@@ -1028,6 +1029,7 @@ static int exec_mmap(struct mm_struct *m tsk->mm->vmacache_seqnum = 0; vmacache_flush(tsk); task_unlock(tsk); @@ -167,8 +165,6 @@ index 881390b44cfd..1afa15a07d26 100644 if (old_mm) { mmap_read_unlock(old_mm); BUG_ON(active_mm != old_mm); -diff --git a/include/linux/memcontrol.h b/include/linux/memcontrol.h -index 8d6a0329bc59..3736405cbcf6 100644 --- a/include/linux/memcontrol.h +++ b/include/linux/memcontrol.h @@ -348,6 +348,11 @@ struct mem_cgroup { @@ -183,8 +179,6 @@ index 8d6a0329bc59..3736405cbcf6 100644 struct mem_cgroup_per_node *nodeinfo[]; }; -diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h -index 7f8ee09c711f..33c142d31261 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -580,6 +580,22 @@ struct mm_struct { @@ -210,7 +204,7 @@ index 7f8ee09c711f..33c142d31261 100644 } __randomize_layout; /* -@@ -606,6 +622,66 @@ static inline cpumask_t *mm_cpumask(struct mm_struct *mm) +@@ -606,6 +622,66 @@ static inline cpumask_t *mm_cpumask(stru return (struct cpumask *)&mm->cpu_bitmap; } @@ -277,8 +271,6 @@ index 7f8ee09c711f..33c142d31261 100644 struct mmu_gather; extern void tlb_gather_mmu(struct mmu_gather *tlb, struct mm_struct *mm); extern void tlb_gather_mmu_fullmm(struct mmu_gather *tlb, struct mm_struct *mm); -diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h -index 4db2b877fcf9..659bab633bdf 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -385,7 +385,7 @@ enum { @@ -365,8 +357,6 @@ index 4db2b877fcf9..659bab633bdf 100644 ZONE_PADDING(_pad2_) /* Per-node vmstats */ -diff --git a/include/linux/swap.h b/include/linux/swap.h -index 4efd267e2937..e970fca4f178 100644 --- a/include/linux/swap.h +++ b/include/linux/swap.h @@ -137,6 +137,10 @@ union swap_header { @@ -380,11 +370,9 @@ index 4efd267e2937..e970fca4f178 100644 }; #ifdef __KERNEL__ -diff --git a/kernel/exit.c b/kernel/exit.c -index 80efdfda6662..06b477395012 100644 --- a/kernel/exit.c +++ b/kernel/exit.c -@@ -469,6 +469,7 @@ void mm_update_next_owner(struct mm_struct *mm) +@@ -469,6 +469,7 @@ assign_new_owner: goto retry; } WRITE_ONCE(mm->owner, c); @@ -392,11 +380,9 @@ index 80efdfda6662..06b477395012 100644 task_unlock(c); put_task_struct(c); } -diff --git a/kernel/fork.c b/kernel/fork.c -index 68eab6ce3085..d8f37ecdde87 100644 --- a/kernel/fork.c +++ b/kernel/fork.c -@@ -1083,6 +1083,7 @@ static struct mm_struct *mm_init(struct mm_struct *mm, struct task_struct *p, +@@ -1083,6 +1083,7 @@ static struct mm_struct *mm_init(struct goto fail_nocontext; mm->user_ns = get_user_ns(user_ns); @@ -404,7 +390,7 @@ index 68eab6ce3085..d8f37ecdde87 100644 return mm; fail_nocontext: -@@ -1125,6 +1126,7 @@ static inline void __mmput(struct mm_struct *mm) +@@ -1125,6 +1126,7 @@ static inline void __mmput(struct mm_str } if (mm->binfmt) module_put(mm->binfmt->module); @@ -412,7 +398,7 @@ index 68eab6ce3085..d8f37ecdde87 100644 mmdrop(mm); } -@@ -2622,6 +2624,13 @@ pid_t kernel_clone(struct kernel_clone_args *args) +@@ -2622,6 +2624,13 @@ pid_t kernel_clone(struct kernel_clone_a get_task_struct(p); } @@ -426,11 +412,9 @@ index 68eab6ce3085..d8f37ecdde87 100644 wake_up_new_task(p); /* forking complete and child started to run, tell ptracer */ -diff --git a/kernel/sched/core.c b/kernel/sched/core.c -index c1458fa8beb3..fe4d60474d4a 100644 --- a/kernel/sched/core.c +++ b/kernel/sched/core.c -@@ -5007,6 +5007,7 @@ context_switch(struct rq *rq, struct task_struct *prev, +@@ -5007,6 +5007,7 @@ context_switch(struct rq *rq, struct tas * finish_task_switch()'s mmdrop(). */ switch_mm_irqs_off(prev->active_mm, next->mm, next); @@ -438,8 +422,6 @@ index c1458fa8beb3..fe4d60474d4a 100644 if (!prev->mm) { // from kernel /* will mmdrop() in finish_task_switch(). */ -diff --git a/mm/memcontrol.c b/mm/memcontrol.c -index cc3431c5d9ba..ed87d1256f0e 100644 --- a/mm/memcontrol.c +++ b/mm/memcontrol.c @@ -6212,6 +6212,30 @@ static void mem_cgroup_move_task(void) @@ -473,7 +455,7 @@ index cc3431c5d9ba..ed87d1256f0e 100644 static int seq_puts_memcg_tunable(struct seq_file *m, unsigned long value) { if (value == PAGE_COUNTER_MAX) -@@ -6555,6 +6579,7 @@ struct cgroup_subsys memory_cgrp_subsys = { +@@ -6555,6 +6579,7 @@ struct cgroup_subsys memory_cgrp_subsys .css_reset = mem_cgroup_css_reset, .css_rstat_flush = mem_cgroup_css_rstat_flush, .can_attach = mem_cgroup_can_attach, @@ -481,8 +463,6 @@ index cc3431c5d9ba..ed87d1256f0e 100644 .cancel_attach = mem_cgroup_cancel_attach, .post_attach = mem_cgroup_move_task, .dfl_cftypes = memory_files, -diff --git a/mm/vmscan.c b/mm/vmscan.c -index 1d0b25ae378c..a7844c689522 100644 --- a/mm/vmscan.c +++ b/mm/vmscan.c @@ -50,6 +50,8 @@ @@ -494,7 +474,7 @@ index 1d0b25ae378c..a7844c689522 100644 #include <asm/tlbflush.h> #include <asm/div64.h> -@@ -2853,7 +2855,7 @@ static bool can_age_anon_pages(struct pglist_data *pgdat, +@@ -2853,7 +2855,7 @@ static bool can_age_anon_pages(struct pg for ((type) = 0; (type) < ANON_AND_FILE; (type)++) \ for ((zone) = 0; (zone) < MAX_NR_ZONES; (zone)++) @@ -503,11 +483,10 @@ index 1d0b25ae378c..a7844c689522 100644 { struct pglist_data *pgdat = NODE_DATA(nid); -@@ -2898,6 +2900,371 @@ static bool __maybe_unused seq_is_valid(struct lruvec *lruvec) - get_nr_gens(lruvec, LRU_GEN_ANON) <= MAX_NR_GENS; +@@ -2899,6 +2901,371 @@ static bool __maybe_unused seq_is_valid( } -+/****************************************************************************** + /****************************************************************************** + * mm_struct list + ******************************************************************************/ + @@ -872,10 +851,11 @@ index 1d0b25ae378c..a7844c689522 100644 + return success; +} + - /****************************************************************************** ++/****************************************************************************** * refault feedback loop ******************************************************************************/ -@@ -3048,6 +3415,118 @@ static int page_inc_gen(struct lruvec *lruvec, struct page *page, bool reclaimin + +@@ -3048,6 +3415,118 @@ static int page_inc_gen(struct lruvec *l return new_gen; } @@ -994,7 +974,7 @@ index 1d0b25ae378c..a7844c689522 100644 static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned long addr) { unsigned long pfn = pte_pfn(pte); -@@ -3066,8 +3545,28 @@ static unsigned long get_pte_pfn(pte_t pte, struct vm_area_struct *vma, unsigned +@@ -3066,8 +3545,28 @@ static unsigned long get_pte_pfn(pte_t p return pfn; } @@ -1024,7 +1004,7 @@ index 1d0b25ae378c..a7844c689522 100644 { struct page *page; -@@ -3082,9 +3581,375 @@ static struct page *get_pfn_page(unsigned long pfn, struct mem_cgroup *memcg, +@@ -3082,9 +3581,375 @@ static struct page *get_pfn_page(unsigne if (page_memcg_rcu(page) != memcg) return NULL; @@ -1400,7 +1380,7 @@ index 1d0b25ae378c..a7844c689522 100644 static void inc_min_seq(struct lruvec *lruvec, int type) { struct lru_gen_struct *lrugen = &lruvec->lrugen; -@@ -3136,7 +4001,7 @@ static bool try_to_inc_min_seq(struct lruvec *lruvec, bool can_swap) +@@ -3136,7 +4001,7 @@ next: return success; } @@ -1409,7 +1389,7 @@ index 1d0b25ae378c..a7844c689522 100644 { int prev, next; int type, zone; -@@ -3146,9 +4011,6 @@ static void inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, bool can_s +@@ -3146,9 +4011,6 @@ static void inc_max_seq(struct lruvec *l VM_WARN_ON_ONCE(!seq_is_valid(lruvec)); @@ -1419,7 +1399,7 @@ index 1d0b25ae378c..a7844c689522 100644 for (type = ANON_AND_FILE - 1; type >= 0; type--) { if (get_nr_gens(lruvec, type) != MAX_NR_GENS) continue; -@@ -3186,10 +4048,76 @@ static void inc_max_seq(struct lruvec *lruvec, unsigned long max_seq, bool can_s +@@ -3186,10 +4048,76 @@ static void inc_max_seq(struct lruvec *l /* make sure preceding modifications appear */ smp_store_release(&lrugen->max_seq, lrugen->max_seq + 1); @@ -1497,7 +1477,7 @@ index 1d0b25ae378c..a7844c689522 100644 static bool should_run_aging(struct lruvec *lruvec, unsigned long max_seq, unsigned long *min_seq, struct scan_control *sc, bool can_swap, unsigned long *nr_to_scan) { -@@ -3265,7 +4193,7 @@ static void age_lruvec(struct lruvec *lruvec, struct scan_control *sc) +@@ -3265,7 +4193,7 @@ static void age_lruvec(struct lruvec *lr need_aging = should_run_aging(lruvec, max_seq, min_seq, sc, swappiness, &nr_to_scan); if (need_aging) @@ -1506,7 +1486,7 @@ index 1d0b25ae378c..a7844c689522 100644 } static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) -@@ -3274,6 +4202,8 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +@@ -3274,6 +4202,8 @@ static void lru_gen_age_node(struct pgli VM_WARN_ON_ONCE(!current_is_kswapd()); @@ -1515,7 +1495,7 @@ index 1d0b25ae378c..a7844c689522 100644 memcg = mem_cgroup_iter(NULL, NULL, NULL); do { struct lruvec *lruvec = mem_cgroup_lruvec(memcg, pgdat); -@@ -3282,11 +4212,16 @@ static void lru_gen_age_node(struct pglist_data *pgdat, struct scan_control *sc) +@@ -3282,11 +4212,16 @@ static void lru_gen_age_node(struct pgli cond_resched(); } while ((memcg = mem_cgroup_iter(NULL, memcg, NULL))); @@ -1533,7 +1513,7 @@ index 1d0b25ae378c..a7844c689522 100644 */ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) { -@@ -3295,6 +4230,8 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +@@ -3295,6 +4230,8 @@ void lru_gen_look_around(struct page_vma unsigned long start; unsigned long end; unsigned long addr; @@ -1542,7 +1522,7 @@ index 1d0b25ae378c..a7844c689522 100644 unsigned long bitmap[BITS_TO_LONGS(MIN_LRU_BATCH)] = {}; struct page *page = pvmw->page; struct mem_cgroup *memcg = page_memcg(page); -@@ -3309,6 +4246,9 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +@@ -3309,6 +4246,9 @@ void lru_gen_look_around(struct page_vma if (spin_is_contended(pvmw->ptl)) return; @@ -1552,7 +1532,7 @@ index 1d0b25ae378c..a7844c689522 100644 start = max(pvmw->address & PMD_MASK, pvmw->vma->vm_start); end = min(pvmw->address | ~PMD_MASK, pvmw->vma->vm_end - 1) + 1; -@@ -3338,13 +4278,15 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +@@ -3338,13 +4278,15 @@ void lru_gen_look_around(struct page_vma if (!pte_young(pte[i])) continue; @@ -1569,7 +1549,7 @@ index 1d0b25ae378c..a7844c689522 100644 if (pte_dirty(pte[i]) && !PageDirty(page) && !(PageAnon(page) && PageSwapBacked(page) && !PageSwapCache(page))) -@@ -3360,7 +4302,11 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +@@ -3360,7 +4302,11 @@ void lru_gen_look_around(struct page_vma arch_leave_lazy_mmu_mode(); rcu_read_unlock(); @@ -1582,7 +1562,7 @@ index 1d0b25ae378c..a7844c689522 100644 for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { page = pte_page(pte[i]); activate_page(page); -@@ -3372,8 +4318,10 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +@@ -3372,8 +4318,10 @@ void lru_gen_look_around(struct page_vma if (!mem_cgroup_trylock_pages(memcg)) return; @@ -1595,7 +1575,7 @@ index 1d0b25ae378c..a7844c689522 100644 for_each_set_bit(i, bitmap, MIN_LRU_BATCH) { page = compound_head(pte_page(pte[i])); -@@ -3384,10 +4332,14 @@ void lru_gen_look_around(struct page_vma_mapped_walk *pvmw) +@@ -3384,10 +4332,14 @@ void lru_gen_look_around(struct page_vma if (old_gen < 0 || old_gen == new_gen) continue; @@ -1612,7 +1592,7 @@ index 1d0b25ae378c..a7844c689522 100644 mem_cgroup_unlock_pages(); } -@@ -3670,6 +4622,7 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp +@@ -3670,6 +4622,7 @@ static int evict_pages(struct lruvec *lr struct page *page; enum vm_event_item item; struct reclaim_stat stat; @@ -1620,7 +1600,7 @@ index 1d0b25ae378c..a7844c689522 100644 struct mem_cgroup *memcg = lruvec_memcg(lruvec); struct pglist_data *pgdat = lruvec_pgdat(lruvec); -@@ -3706,6 +4659,10 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp +@@ -3706,6 +4659,10 @@ static int evict_pages(struct lruvec *lr move_pages_to_lru(lruvec, &list); @@ -1631,7 +1611,7 @@ index 1d0b25ae378c..a7844c689522 100644 item = current_is_kswapd() ? PGSTEAL_KSWAPD : PGSTEAL_DIRECT; if (!cgroup_reclaim(sc)) __count_vm_events(item, reclaimed); -@@ -3722,6 +4679,11 @@ static int evict_pages(struct lruvec *lruvec, struct scan_control *sc, int swapp +@@ -3722,6 +4679,11 @@ static int evict_pages(struct lruvec *lr return scanned; } @@ -1643,7 +1623,7 @@ index 1d0b25ae378c..a7844c689522 100644 static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control *sc, bool can_swap) { -@@ -3747,7 +4709,8 @@ static unsigned long get_nr_to_scan(struct lruvec *lruvec, struct scan_control * +@@ -3747,7 +4709,8 @@ static unsigned long get_nr_to_scan(stru if (current_is_kswapd()) return 0; @@ -1653,7 +1633,7 @@ index 1d0b25ae378c..a7844c689522 100644 done: return min_seq[!can_swap] + MIN_NR_GENS <= max_seq ? nr_to_scan : 0; } -@@ -3761,6 +4724,8 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc +@@ -3761,6 +4724,8 @@ static void lru_gen_shrink_lruvec(struct blk_start_plug(&plug); @@ -1662,7 +1642,7 @@ index 1d0b25ae378c..a7844c689522 100644 while (true) { int delta; int swappiness; -@@ -3788,6 +4753,8 @@ static void lru_gen_shrink_lruvec(struct lruvec *lruvec, struct scan_control *sc +@@ -3788,6 +4753,8 @@ static void lru_gen_shrink_lruvec(struct cond_resched(); } @@ -1671,7 +1651,7 @@ index 1d0b25ae378c..a7844c689522 100644 blk_finish_plug(&plug); } -@@ -3804,15 +4771,21 @@ void lru_gen_init_lruvec(struct lruvec *lruvec) +@@ -3804,15 +4771,21 @@ void lru_gen_init_lruvec(struct lruvec * for_each_gen_type_zone(gen, type, zone) INIT_LIST_HEAD(&lrugen->lists[gen][type][zone]); @@ -1693,7 +1673,7 @@ index 1d0b25ae378c..a7844c689522 100644 int nid; for_each_node(nid) { -@@ -3820,6 +4793,11 @@ void lru_gen_exit_memcg(struct mem_cgroup *memcg) +@@ -3820,6 +4793,11 @@ void lru_gen_exit_memcg(struct mem_cgrou VM_WARN_ON_ONCE(memchr_inv(lruvec->lrugen.nr_pages, 0, sizeof(lruvec->lrugen.nr_pages))); @@ -1705,6 +1685,3 @@ index 1d0b25ae378c..a7844c689522 100644 } } #endif --- -2.40.0 - |