1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
|
/* SPDX-License-Identifier: GPL-2.0 */
/*
* Kernel Electric-Fence (KFENCE). For more info please see
* Documentation/dev-tools/kfence.rst.
*
* Copyright (C) 2020, Google LLC.
*/
#ifndef MM_KFENCE_KFENCE_H
#define MM_KFENCE_KFENCE_H
#include <linux/mm.h>
#include <linux/slab.h>
#include <linux/spinlock.h>
#include <linux/types.h>
#include "../slab.h" /* for struct kmem_cache */
/*
* Get the canary byte pattern for @addr. Use a pattern that varies based on the
* lower 3 bits of the address, to detect memory corruptions with higher
* probability, where similar constants are used.
*/
#define KFENCE_CANARY_PATTERN_U8(addr) ((u8)0xaa ^ (u8)((unsigned long)(addr) & 0x7))
/*
* Define a continuous 8-byte canary starting from a multiple of 8. The canary
* of each byte is only related to the lowest three bits of its address, so the
* canary of every 8 bytes is the same. 64-bit memory can be filled and checked
* at a time instead of byte by byte to improve performance.
*/
#define KFENCE_CANARY_PATTERN_U64 ((u64)0xaaaaaaaaaaaaaaaa ^ (u64)(le64_to_cpu(0x0706050403020100)))
/* Maximum stack depth for reports. */
#define KFENCE_STACK_DEPTH 64
/* KFENCE object states. */
enum kfence_object_state {
KFENCE_OBJECT_UNUSED, /* Object is unused. */
KFENCE_OBJECT_ALLOCATED, /* Object is currently allocated. */
KFENCE_OBJECT_RCU_FREEING, /* Object was allocated, and then being freed by rcu. */
KFENCE_OBJECT_FREED, /* Object was allocated, and then freed. */
};
/* Alloc/free tracking information. */
struct kfence_track {
pid_t pid;
int cpu;
u64 ts_nsec;
int num_stack_entries;
unsigned long stack_entries[KFENCE_STACK_DEPTH];
};
/* KFENCE metadata per guarded allocation. */
struct kfence_metadata {
struct list_head list; /* Freelist node; access under kfence_freelist_lock. */
struct rcu_head rcu_head; /* For delayed freeing. */
/*
* Lock protecting below data; to ensure consistency of the below data,
* since the following may execute concurrently: __kfence_alloc(),
* __kfence_free(), kfence_handle_page_fault(). However, note that we
* cannot grab the same metadata off the freelist twice, and multiple
* __kfence_alloc() cannot run concurrently on the same metadata.
*/
raw_spinlock_t lock;
/* The current state of the object; see above. */
enum kfence_object_state state;
/*
* Allocated object address; cannot be calculated from size, because of
* alignment requirements.
*
* Invariant: ALIGN_DOWN(addr, PAGE_SIZE) is constant.
*/
unsigned long addr;
/*
* The size of the original allocation.
*/
size_t size;
/*
* The kmem_cache cache of the last allocation; NULL if never allocated
* or the cache has already been destroyed.
*/
struct kmem_cache *cache;
/*
* In case of an invalid access, the page that was unprotected; we
* optimistically only store one address.
*/
unsigned long unprotected_page;
/* Allocation and free stack information. */
struct kfence_track alloc_track;
struct kfence_track free_track;
/* For updating alloc_covered on frees. */
u32 alloc_stack_hash;
#ifdef CONFIG_MEMCG
struct slabobj_ext obj_exts;
#endif
};
#define KFENCE_METADATA_SIZE PAGE_ALIGN(sizeof(struct kfence_metadata) * \
CONFIG_KFENCE_NUM_OBJECTS)
extern struct kfence_metadata *kfence_metadata;
static inline struct kfence_metadata *addr_to_metadata(unsigned long addr)
{
long index;
/* The checks do not affect performance; only called from slow-paths. */
if (!is_kfence_address((void *)addr))
return NULL;
/*
* May be an invalid index if called with an address at the edge of
* __kfence_pool, in which case we would report an "invalid access"
* error.
*/
index = (addr - (unsigned long)__kfence_pool) / (PAGE_SIZE * 2) - 1;
if (index < 0 || index >= CONFIG_KFENCE_NUM_OBJECTS)
return NULL;
return &kfence_metadata[index];
}
/* KFENCE error types for report generation. */
enum kfence_error_type {
KFENCE_ERROR_OOB, /* Detected a out-of-bounds access. */
KFENCE_ERROR_UAF, /* Detected a use-after-free access. */
KFENCE_ERROR_CORRUPTION, /* Detected a memory corruption on free. */
KFENCE_ERROR_INVALID, /* Invalid access of unknown type. */
KFENCE_ERROR_INVALID_FREE, /* Invalid free. */
};
void kfence_report_error(unsigned long address, bool is_write, struct pt_regs *regs,
const struct kfence_metadata *meta, enum kfence_error_type type);
void kfence_print_object(struct seq_file *seq, const struct kfence_metadata *meta);
#endif /* MM_KFENCE_KFENCE_H */
|
