From: Andrey Konovalov andreyknvl@google.com
mainline inclusion from mainline-v5.11-rc1 commit 97593cad003c668e2532cb2939a24a031f8de52d category: bugfix bugzilla: 187796, https://gitee.com/openeuler/kernel/issues/I5W6YV CVE: NA
Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i...
--------------------------------
KASAN marks caches that are sanitized with the SLAB_KASAN cache flag. Currently if the metadata that is appended after the object (stores e.g. stack trace ids) doesn't fit into KMALLOC_MAX_SIZE (can only happen with SLAB, see the comment in the patch), KASAN turns off sanitization completely.
With this change sanitization of the object data is always enabled. However the metadata is only stored when it fits. Instead of checking for SLAB_KASAN flag accross the code to find out whether the metadata is there, use cache->kasan_info.alloc/free_meta_offset. As 0 can be a valid value for free_meta_offset, introduce KASAN_NO_FREE_META as an indicator that the free metadata is missing.
Without this change all sanitized KASAN objects would be put into quarantine with generic KASAN. With this change, only the objects that have metadata (i.e. when it fits) are put into quarantine, the rest is freed right away.
Along the way rework __kasan_cache_create() and add claryfying comments.
Link: https://lkml.kernel.org/r/aee34b87a5e4afe586c2ac6a0b32db8dc4dcc2dc.160616239... Link: https://linux-review.googlesource.com/id/Icd947e2bea054cb5cfbdc6cf6652227d97... Co-developed-by: Vincenzo Frascino Vincenzo.Frascino@arm.com Signed-off-by: Vincenzo Frascino Vincenzo.Frascino@arm.com Signed-off-by: Andrey Konovalov andreyknvl@google.com Reviewed-by: Marco Elver elver@google.com Tested-by: Vincenzo Frascino vincenzo.frascino@arm.com Cc: Alexander Potapenko glider@google.com Cc: Andrey Ryabinin aryabinin@virtuozzo.com Cc: Branislav Rankov Branislav.Rankov@arm.com Cc: Catalin Marinas catalin.marinas@arm.com Cc: Dmitry Vyukov dvyukov@google.com Cc: Evgenii Stepanov eugenis@google.com Cc: Kevin Brodsky kevin.brodsky@arm.com Cc: Vasily Gorbik gor@linux.ibm.com Cc: Will Deacon will.deacon@arm.com Signed-off-by: Andrew Morton akpm@linux-foundation.org Signed-off-by: Linus Torvalds torvalds@linux-foundation.org Conflicts: mm/kasan/common.c mm/kasan/hw_tags.c mm/kasan/kasan.h mm/kasan/quarantine.c mm/kasan/report.c mm/kasan/report_sw_tags.c mm/kasan/sw_tags.c Signed-off-by: Liu Shixin liushixin2@huawei.com Reviewed-by: Kefeng Wang wangkefeng.wang@huawei.com Signed-off-by: Zheng Zengkai zhengzengkai@huawei.com --- mm/kasan/common.c | 113 ++++++++++++++++++++++++++--------------- mm/kasan/generic.c | 9 ++-- mm/kasan/kasan.h | 17 +++++-- mm/kasan/quarantine.c | 18 ++++++- mm/kasan/report.c | 43 ++++++++-------- mm/kasan/tags.c | 4 ++ mm/kasan/tags_report.c | 9 ++-- 7 files changed, 141 insertions(+), 72 deletions(-)
diff --git a/mm/kasan/common.c b/mm/kasan/common.c index 03e679121640..84c8d626d37d 100644 --- a/mm/kasan/common.c +++ b/mm/kasan/common.c @@ -220,9 +220,6 @@ void kasan_free_pages(struct page *page, unsigned int order) */ static inline unsigned int optimal_redzone(unsigned int object_size) { - if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) - return 0; - return object_size <= 64 - 16 ? 16 : object_size <= 128 - 32 ? 32 : @@ -236,42 +233,74 @@ static inline unsigned int optimal_redzone(unsigned int object_size) void kasan_cache_create(struct kmem_cache *cache, unsigned int *size, slab_flags_t *flags) { - unsigned int orig_size = *size; - unsigned int redzone_size; - int redzone_adjust; + unsigned int ok_size; + unsigned int optimal_size;
- /* Add alloc meta. */ + /* + * SLAB_KASAN is used to mark caches as ones that are sanitized by + * KASAN. Currently this flag is used in two places: + * 1. In slab_ksize() when calculating the size of the accessible + * memory within the object. + * 2. In slab_common.c to prevent merging of sanitized caches. + */ + *flags |= SLAB_KASAN; + + ok_size = *size; + + /* Add alloc meta into redzone. */ cache->kasan_info.alloc_meta_offset = *size; *size += sizeof(struct kasan_alloc_meta);
- /* Add free meta. */ - if (IS_ENABLED(CONFIG_KASAN_GENERIC) && - (cache->flags & SLAB_TYPESAFE_BY_RCU || cache->ctor || - cache->object_size < sizeof(struct kasan_free_meta))) { - cache->kasan_info.free_meta_offset = *size; - *size += sizeof(struct kasan_free_meta); + /* + * If alloc meta doesn't fit, don't add it. + * This can only happen with SLAB, as it has KMALLOC_MAX_SIZE equal + * to KMALLOC_MAX_CACHE_SIZE and doesn't fall back to page_alloc for + * larger sizes. + */ + if (*size > KMALLOC_MAX_SIZE) { + cache->kasan_info.alloc_meta_offset = 0; + *size = ok_size; + /* Continue, since free meta might still fit. */ }
- redzone_size = optimal_redzone(cache->object_size); - redzone_adjust = redzone_size - (*size - cache->object_size); - if (redzone_adjust > 0) - *size += redzone_adjust; - - *size = min_t(unsigned int, KMALLOC_MAX_SIZE, - max(*size, cache->object_size + redzone_size)); + /* Only the generic mode uses free meta or flexible redzones. */ + if (!IS_ENABLED(CONFIG_KASAN_GENERIC)) { + cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META; + return; + }
/* - * If the metadata doesn't fit, don't enable KASAN at all. + * Add free meta into redzone when it's not possible to store + * it in the object. This is the case when: + * 1. Object is SLAB_TYPESAFE_BY_RCU, which means that it can + * be touched after it was freed, or + * 2. Object has a constructor, which means it's expected to + * retain its content until the next allocation, or + * 3. Object is too small. + * Otherwise cache->kasan_info.free_meta_offset = 0 is implied. */ - if (*size <= cache->kasan_info.alloc_meta_offset || - *size <= cache->kasan_info.free_meta_offset) { - cache->kasan_info.alloc_meta_offset = 0; - cache->kasan_info.free_meta_offset = 0; - *size = orig_size; - return; + if ((cache->flags & SLAB_TYPESAFE_BY_RCU) || cache->ctor || + cache->object_size < sizeof(struct kasan_free_meta)) { + ok_size = *size; + + cache->kasan_info.free_meta_offset = *size; + *size += sizeof(struct kasan_free_meta); + + /* If free meta doesn't fit, don't add it. */ + if (*size > KMALLOC_MAX_SIZE) { + cache->kasan_info.free_meta_offset = KASAN_NO_FREE_META; + *size = ok_size; + } }
- *flags |= SLAB_KASAN; + /* Calculate size with optimal redzone. */ + optimal_size = cache->object_size + optimal_redzone(cache->object_size); + /* Limit it with KMALLOC_MAX_SIZE (relevant for SLAB only). */ + if (optimal_size > KMALLOC_MAX_SIZE) + optimal_size = KMALLOC_MAX_SIZE; + /* Use optimal size if the size with added metas is not large enough. */ + if (*size < optimal_size) + *size = optimal_size; }
size_t kasan_metadata_size(struct kmem_cache *cache) @@ -285,15 +314,21 @@ size_t kasan_metadata_size(struct kmem_cache *cache) struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, const void *object) { + if (!cache->kasan_info.alloc_meta_offset) + return NULL; return (void *)object + cache->kasan_info.alloc_meta_offset; }
+#ifdef CONFIG_KASAN_GENERIC struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, const void *object) { BUILD_BUG_ON(sizeof(struct kasan_free_meta) > 32); + if (cache->kasan_info.free_meta_offset == KASAN_NO_FREE_META) + return NULL; return (void *)object + cache->kasan_info.free_meta_offset; } +#endif
void kasan_poison_slab(struct page *page) { @@ -368,11 +403,9 @@ void * __must_check kasan_init_slab_obj(struct kmem_cache *cache, { struct kasan_alloc_meta *alloc_meta;
- if (!(cache->flags & SLAB_KASAN)) - return (void *)object; - alloc_meta = kasan_get_alloc_meta(cache, object); - __memset(alloc_meta, 0, sizeof(*alloc_meta)); + if (alloc_meta) + __memset(alloc_meta, 0, sizeof(*alloc_meta));
if (IS_ENABLED(CONFIG_KASAN_SW_TAGS)) object = set_tag(object, @@ -430,15 +463,12 @@ static bool __kasan_slab_free(struct kmem_cache *cache, void *object, rounded_up_size = round_up(cache->object_size, KASAN_SHADOW_SCALE_SIZE); kasan_poison_shadow(object, rounded_up_size, KASAN_KMALLOC_FREE);
- if ((IS_ENABLED(CONFIG_KASAN_GENERIC) && !quarantine) || - unlikely(!(cache->flags & SLAB_KASAN))) + if ((IS_ENABLED(CONFIG_KASAN_GENERIC) && !quarantine)) return false;
kasan_set_free_info(cache, object, tag);
- quarantine_put(cache, object); - - return IS_ENABLED(CONFIG_KASAN_GENERIC); + return quarantine_put(cache, object); }
bool kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip) @@ -448,7 +478,11 @@ bool kasan_slab_free(struct kmem_cache *cache, void *object, unsigned long ip)
static void set_alloc_info(struct kmem_cache *cache, void *object, gfp_t flags) { - kasan_set_track(&kasan_get_alloc_meta(cache, object)->alloc_track, flags); + struct kasan_alloc_meta *alloc_meta; + + alloc_meta = kasan_get_alloc_meta(cache, object); + if (alloc_meta) + kasan_set_track(&alloc_meta->alloc_track, flags); }
static void *__kasan_kmalloc(struct kmem_cache *cache, const void *object, @@ -480,8 +514,7 @@ static void *__kasan_kmalloc(struct kmem_cache *cache, const void *object, kasan_poison_shadow((void *)redzone_start, redzone_end - redzone_start, KASAN_KMALLOC_REDZONE);
- if (cache->flags & SLAB_KASAN) - set_alloc_info(cache, (void *)object, flags); + set_alloc_info(cache, (void *)object, flags);
return set_tag(object, tag); } diff --git a/mm/kasan/generic.c b/mm/kasan/generic.c index 52a1383512c7..671f7c233896 100644 --- a/mm/kasan/generic.c +++ b/mm/kasan/generic.c @@ -355,11 +355,11 @@ void kasan_set_free_info(struct kmem_cache *cache, struct kasan_free_meta *free_meta;
free_meta = kasan_get_free_meta(cache, object); - kasan_set_track(&free_meta->free_track, GFP_NOWAIT); + if (!free_meta) + return;
- /* - * the object was freed and has free track set - */ + kasan_set_track(&free_meta->free_track, GFP_NOWAIT); + /* The object was freed and has free track set. */ *(u8 *)kasan_mem_to_shadow(object) = KASAN_KMALLOC_FREETRACK; }
@@ -368,5 +368,6 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, { if (*(u8 *)kasan_mem_to_shadow(object) != KASAN_KMALLOC_FREETRACK) return NULL; + /* Free meta must be present with KASAN_KMALLOC_FREETRACK. */ return &kasan_get_free_meta(cache, object)->free_track; } diff --git a/mm/kasan/kasan.h b/mm/kasan/kasan.h index 8e9fab29627c..17d97ffbbdeb 100644 --- a/mm/kasan/kasan.h +++ b/mm/kasan/kasan.h @@ -124,20 +124,31 @@ struct kasan_alloc_meta { struct qlist_node { struct qlist_node *next; }; + +/* + * Generic mode either stores free meta in the object itself or in the redzone + * after the object. In the former case free meta offset is 0, in the latter + * case it has some sane value smaller than INT_MAX. Use INT_MAX as free meta + * offset when free meta isn't present. + */ +#define KASAN_NO_FREE_META INT_MAX + struct kasan_free_meta { +#ifdef CONFIG_KASAN_GENERIC /* This field is used while the object is in the quarantine. * Otherwise it might be used for the allocator freelist. */ struct qlist_node quarantine_link; -#ifdef CONFIG_KASAN_GENERIC struct kasan_track free_track; #endif };
struct kasan_alloc_meta *kasan_get_alloc_meta(struct kmem_cache *cache, const void *object); +#ifdef CONFIG_KASAN_GENERIC struct kasan_free_meta *kasan_get_free_meta(struct kmem_cache *cache, const void *object); +#endif
static inline const void *kasan_shadow_to_mem(const void *shadow_addr) { @@ -180,11 +191,11 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache,
#if defined(CONFIG_KASAN_GENERIC) && \ (defined(CONFIG_SLAB) || defined(CONFIG_SLUB)) -void quarantine_put(struct kmem_cache *cache, void *object); +bool quarantine_put(struct kmem_cache *cache, void *object); void quarantine_reduce(void); void quarantine_remove_cache(struct kmem_cache *cache); #else -static inline void quarantine_put(struct kmem_cache *cache, void *object) { } +static inline void quarantine_put(struct kmem_cache *cache, void *object) { return false; } static inline void quarantine_reduce(void) { } static inline void quarantine_remove_cache(struct kmem_cache *cache) { } #endif diff --git a/mm/kasan/quarantine.c b/mm/kasan/quarantine.c index ed0bebbe61c6..12cfc55dbc2d 100644 --- a/mm/kasan/quarantine.c +++ b/mm/kasan/quarantine.c @@ -147,7 +147,12 @@ static void qlink_free(struct qlist_node *qlink, struct kmem_cache *cache) if (IS_ENABLED(CONFIG_SLAB)) local_irq_save(flags);
+ /* + * As the object now gets freed from the quaratine, assume that its + * free track is no longer valid. + */ *(u8 *)kasan_mem_to_shadow(object) = KASAN_KMALLOC_FREE; + ___cache_free(cache, object, _THIS_IP_);
if (IS_ENABLED(CONFIG_SLAB)) @@ -173,13 +178,20 @@ static void qlist_free_all(struct qlist_head *q, struct kmem_cache *cache) qlist_init(q); }
-void quarantine_put(struct kmem_cache *cache, void *object) +bool quarantine_put(struct kmem_cache *cache, void *object) { unsigned long flags; struct qlist_head *q; struct qlist_head temp = QLIST_INIT; struct kasan_free_meta *meta = kasan_get_free_meta(cache, object);
+ /* + * If there's no metadata for this object, don't put it into + * quarantine. + */ + if (!meta) + return false; + /* * Note: irq must be disabled until after we move the batch to the * global quarantine. Otherwise quarantine_remove_cache() can miss @@ -193,7 +205,7 @@ void quarantine_put(struct kmem_cache *cache, void *object) q = this_cpu_ptr(&cpu_quarantine); if (q->offline) { local_irq_restore(flags); - return; + return false; } qlist_put(q, &meta->quarantine_link, cache->size); if (unlikely(q->bytes > QUARANTINE_PERCPU_SIZE)) { @@ -216,6 +228,8 @@ void quarantine_put(struct kmem_cache *cache, void *object) }
local_irq_restore(flags); + + return true; }
void quarantine_reduce(void) diff --git a/mm/kasan/report.c b/mm/kasan/report.c index cf080b816b47..1cb45431da7d 100644 --- a/mm/kasan/report.c +++ b/mm/kasan/report.c @@ -172,32 +172,35 @@ static void describe_object_addr(struct kmem_cache *cache, void *object, static void describe_object(struct kmem_cache *cache, void *object, const void *addr, u8 tag) { - struct kasan_alloc_meta *alloc_meta = kasan_get_alloc_meta(cache, object); - - if (cache->flags & SLAB_KASAN) { - struct kasan_track *free_track; + struct kasan_alloc_meta *alloc_meta; + struct kasan_track *free_track;
+ alloc_meta = kasan_get_alloc_meta(cache, object); + if (alloc_meta) { print_track(&alloc_meta->alloc_track, "Allocated"); pr_err("\n"); - free_track = kasan_get_free_track(cache, object, tag); - if (free_track) { - print_track(free_track, "Freed"); - pr_err("\n"); - } + } + + free_track = kasan_get_free_track(cache, object, tag); + if (free_track) { + print_track(free_track, "Freed"); + pr_err("\n"); + }
#ifdef CONFIG_KASAN_GENERIC - if (alloc_meta->aux_stack[0]) { - pr_err("Last call_rcu():\n"); - print_stack(alloc_meta->aux_stack[0]); - pr_err("\n"); - } - if (alloc_meta->aux_stack[1]) { - pr_err("Second to last call_rcu():\n"); - print_stack(alloc_meta->aux_stack[1]); - pr_err("\n"); - } -#endif + if (!alloc_meta) + return; + if (alloc_meta->aux_stack[0]) { + pr_err("Last call_rcu():\n"); + print_stack(alloc_meta->aux_stack[0]); + pr_err("\n"); } + if (alloc_meta->aux_stack[1]) { + pr_err("Second to last call_rcu():\n"); + print_stack(alloc_meta->aux_stack[1]); + pr_err("\n"); + } +#endif
describe_object_addr(cache, object, addr); } diff --git a/mm/kasan/tags.c b/mm/kasan/tags.c index 5b4ac4ddfbfc..ce401d454418 100644 --- a/mm/kasan/tags.c +++ b/mm/kasan/tags.c @@ -169,6 +169,8 @@ void kasan_set_free_info(struct kmem_cache *cache, u8 idx = 0;
alloc_meta = kasan_get_alloc_meta(cache, object); + if (!alloc_meta) + return;
#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY idx = alloc_meta->free_track_idx; @@ -186,6 +188,8 @@ struct kasan_track *kasan_get_free_track(struct kmem_cache *cache, int i = 0;
alloc_meta = kasan_get_alloc_meta(cache, object); + if (!alloc_meta) + return NULL;
#ifdef CONFIG_KASAN_SW_TAGS_IDENTIFY for (i = 0; i < KASAN_NR_FREE_STACKS; i++) { diff --git a/mm/kasan/tags_report.c b/mm/kasan/tags_report.c index 5fb0e54a2ad8..157588e89e60 100644 --- a/mm/kasan/tags_report.c +++ b/mm/kasan/tags_report.c @@ -53,9 +53,12 @@ const char *get_bug_type(struct kasan_access_info *info) object = nearest_obj(cache, page, (void *)addr); alloc_meta = kasan_get_alloc_meta(cache, object);
- for (i = 0; i < KASAN_NR_FREE_STACKS; i++) - if (alloc_meta->free_pointer_tag[i] == tag) - return "use-after-free"; + if (alloc_meta) { + for (i = 0; i < KASAN_NR_FREE_STACKS; i++) { + if (alloc_meta->free_pointer_tag[i] == tag) + return "use-after-free"; + } + } return "out-of-bounds"; }