diff --git a/include/linux/mmzone.h b/include/linux/mmzone.h index 769bab28dc1c..3394901d0cf6 100644 --- a/include/linux/mmzone.h +++ b/include/linux/mmzone.h @@ -40,8 +40,6 @@ enum migratetype { MIGRATE_UNMOVABLE, MIGRATE_MOVABLE, MIGRATE_RECLAIMABLE, - MIGRATE_PCPTYPES, /* the number of types on the pcp lists */ - MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES, #ifdef CONFIG_CMA /* * MIGRATE_CMA migration type is designed to mimic the way @@ -58,6 +56,8 @@ enum migratetype { */ MIGRATE_CMA, #endif + MIGRATE_PCPTYPES, /* the number of types on the pcp lists */ + MIGRATE_HIGHATOMIC = MIGRATE_PCPTYPES, #ifdef CONFIG_MEMORY_ISOLATION MIGRATE_ISOLATE, /* can't allocate from here */ #endif @@ -70,9 +70,11 @@ extern char * const migratetype_names[MIGRATE_TYPES]; #ifdef CONFIG_CMA # define is_migrate_cma(migratetype) unlikely((migratetype) == MIGRATE_CMA) # define is_migrate_cma_page(_page) (get_pageblock_migratetype(_page) == MIGRATE_CMA) +# define get_cma_migrate_type() MIGRATE_CMA #else # define is_migrate_cma(migratetype) false # define is_migrate_cma_page(_page) false +# define get_cma_migrate_type() MIGRATE_MOVABLE #endif static inline bool is_migrate_movable(int mt) diff --git a/mm/page_alloc.c b/mm/page_alloc.c index ee41b849d276..b26a1a67fb1a 100644 --- a/mm/page_alloc.c +++ b/mm/page_alloc.c @@ -291,10 +291,10 @@ char * const migratetype_names[MIGRATE_TYPES] = { "Unmovable", "Movable", "Reclaimable", - "HighAtomic", #ifdef CONFIG_CMA "CMA", #endif + "HighAtomic", #ifdef CONFIG_MEMORY_ISOLATION "Isolate", #endif @@ -2588,25 +2588,23 @@ __rmqueue(struct zone *zone, unsigned int order, int migratetype, return page; } -static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, - int migratetype) +#ifdef CONFIG_CMA +static struct page *__rmqueue_cma(struct zone *zone, unsigned int order) { struct page *page = 0; -retry: -#ifdef CONFIG_CMA - if (migratetype == MIGRATE_MOVABLE && !zone->cma_alloc) - page = __rmqueue_cma_fallback(zone, order); - else -#endif - page = __rmqueue_smallest(zone, order, migratetype); - - if (unlikely(!page) && __rmqueue_fallback(zone, order, migratetype)) - goto retry; - - trace_mm_page_alloc_zone_locked(page, order, migratetype); + if (IS_ENABLED(CONFIG_CMA)) + if (!zone->cma_alloc) + page = __rmqueue_cma_fallback(zone, order); + trace_mm_page_alloc_zone_locked(page, order, MIGRATE_CMA); return page; } +#else +static inline struct page *__rmqueue_cma(struct zone *zone, unsigned int order) +{ + return NULL; +} +#endif /* * Obtain a specified number of elements from the buddy allocator, all under @@ -2615,7 +2613,7 @@ static struct page *__rmqueue_cma(struct zone *zone, unsigned int order, */ static int rmqueue_bulk(struct zone *zone, unsigned int order, unsigned long count, struct list_head *list, - int migratetype, unsigned int alloc_flags, int cma) + int migratetype, unsigned int alloc_flags) { int i, alloced = 0; @@ -2623,8 +2621,13 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, for (i = 0; i < count; ++i) { struct page *page; - if (cma) - page = __rmqueue_cma(zone, order, migratetype); + /* + * If migrate type CMA is being requested only try to + * satisfy the request with CMA pages to try and increase + * CMA utlization. + */ + if (is_migrate_cma(migratetype)) + page = __rmqueue_cma(zone, order); else page = __rmqueue(zone, order, migratetype, alloc_flags); @@ -2662,6 +2665,28 @@ static int rmqueue_bulk(struct zone *zone, unsigned int order, return alloced; } +/* + * Return the pcp list that corresponds to the migrate type if that list isn't + * empty. + * If the list is empty return NULL. + */ +static struct list_head *get_populated_pcp_list(struct zone *zone, + unsigned int order, struct per_cpu_pages *pcp, + int migratetype, unsigned int alloc_flags) +{ + struct list_head *list = &pcp->lists[migratetype]; + + if (list_empty(list)) { + pcp->count += rmqueue_bulk(zone, order, + pcp->batch, list, + migratetype, alloc_flags); + + if (list_empty(list)) + list = NULL; + } + return list; +} + #ifdef CONFIG_NUMA /* * Called from the vmstat counter updater to drain pagesets of this @@ -3085,17 +3110,28 @@ static inline void zone_statistics(struct zone *preferred_zone, struct zone *z) static struct page *__rmqueue_pcplist(struct zone *zone, int migratetype, unsigned int alloc_flags, struct per_cpu_pages *pcp, - struct list_head *list, gfp_t gfp_flags) + gfp_t gfp_flags) { - struct page *page; + struct page *page = NULL; + struct list_head *list = NULL; do { - if (list_empty(list)) { - pcp->count += rmqueue_bulk(zone, 0, - pcp->batch, list, - migratetype, alloc_flags, - gfp_flags & __GFP_CMA); - if (unlikely(list_empty(list))) + /* First try to get CMA pages */ + if (migratetype == MIGRATE_MOVABLE && + gfp_flags & __GFP_CMA) { + list = get_populated_pcp_list(zone, 0, pcp, + get_cma_migrate_type(), alloc_flags); + } + + if (list == NULL) { + /* + * Either CMA is not suitable or there are no + * free CMA pages. + */ + list = get_populated_pcp_list(zone, 0, pcp, + migratetype, alloc_flags); + if (unlikely(list == NULL) || + unlikely(list_empty(list))) return NULL; } @@ -3114,14 +3150,12 @@ static struct page *rmqueue_pcplist(struct zone *preferred_zone, unsigned int alloc_flags) { struct per_cpu_pages *pcp; - struct list_head *list; struct page *page; unsigned long flags; local_irq_save(flags); pcp = &this_cpu_ptr(zone->pageset)->pcp; - list = &pcp->lists[migratetype]; - page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, list, + page = __rmqueue_pcplist(zone, migratetype, alloc_flags, pcp, gfp_flags); if (page) { __count_zid_vm_events(PGALLOC, page_zonenum(page), 1 << order); @@ -3158,19 +3192,21 @@ struct page *rmqueue(struct zone *preferred_zone, do { page = NULL; + if (alloc_flags & ALLOC_HARDER) { page = __rmqueue_smallest(zone, order, MIGRATE_HIGHATOMIC); if (page) trace_mm_page_alloc_zone_locked(page, order, migratetype); } - if (!page) { - if (gfp_flags & __GFP_CMA) - page = __rmqueue_cma(zone, order, migratetype); - else - page = __rmqueue(zone, order, migratetype, - alloc_flags); - } + + if (!page && migratetype == MIGRATE_MOVABLE && + gfp_flags & __GFP_CMA) + page = __rmqueue_cma(zone, order); + + if (!page) + page = __rmqueue(zone, order, migratetype, alloc_flags); } while (page && check_new_pages(page, order)); + spin_unlock(&zone->lock); if (!page) goto failed;