Branch data Line data Source code
1 : : // SPDX-License-Identifier: GPL-2.0-only
2 : : /*
3 : : * Access kernel memory without faulting.
4 : : */
5 : : #include <linux/export.h>
6 : : #include <linux/mm.h>
7 : : #include <linux/uaccess.h>
8 : :
9 : : static __always_inline long
10 : 5158218 : probe_read_common(void *dst, const void __user *src, size_t size)
11 : : {
12 : 5158218 : long ret;
13 : :
14 : 5158218 : pagefault_disable();
15 : 5158218 : ret = __copy_from_user_inatomic(dst, src, size);
16 : 5158218 : pagefault_enable();
17 : :
18 [ - - + - ]: 5158218 : return ret ? -EFAULT : 0;
19 : : }
20 : :
21 : : static __always_inline long
22 : 0 : probe_write_common(void __user *dst, const void *src, size_t size)
23 : : {
24 : 0 : long ret;
25 : :
26 : 0 : pagefault_disable();
27 : 0 : ret = __copy_to_user_inatomic(dst, src, size);
28 : 0 : pagefault_enable();
29 : :
30 [ # # # # ]: 0 : return ret ? -EFAULT : 0;
31 : : }
32 : :
33 : : /**
34 : : * probe_kernel_read(): safely attempt to read from a kernel-space location
35 : : * @dst: pointer to the buffer that shall take the data
36 : : * @src: address to read from
37 : : * @size: size of the data chunk
38 : : *
39 : : * Safely read from address @src to the buffer at @dst. If a kernel fault
40 : : * happens, handle that and return -EFAULT.
41 : : *
42 : : * We ensure that the copy_from_user is executed in atomic context so that
43 : : * do_page_fault() doesn't attempt to take mmap_sem. This makes
44 : : * probe_kernel_read() suitable for use within regions where the caller
45 : : * already holds mmap_sem, or other locks which nest inside mmap_sem.
46 : : *
47 : : * probe_kernel_read_strict() is the same as probe_kernel_read() except for
48 : : * the case where architectures have non-overlapping user and kernel address
49 : : * ranges: probe_kernel_read_strict() will additionally return -EFAULT for
50 : : * probing memory on a user address range where probe_user_read() is supposed
51 : : * to be used instead.
52 : : */
53 : :
54 : : long __weak probe_kernel_read(void *dst, const void *src, size_t size)
55 : : __attribute__((alias("__probe_kernel_read")));
56 : :
57 : : long __weak probe_kernel_read_strict(void *dst, const void *src, size_t size)
58 : : __attribute__((alias("__probe_kernel_read")));
59 : :
60 : 5158218 : long __probe_kernel_read(void *dst, const void *src, size_t size)
61 : : {
62 : 5158218 : long ret;
63 : 5158218 : mm_segment_t old_fs = get_fs();
64 : :
65 : 5158218 : set_fs(KERNEL_DS);
66 : 5158218 : ret = probe_read_common(dst, (__force const void __user *)src, size);
67 : 5158218 : set_fs(old_fs);
68 : :
69 : 5158218 : return ret;
70 : : }
71 : : EXPORT_SYMBOL_GPL(probe_kernel_read);
72 : :
73 : : /**
74 : : * probe_user_read(): safely attempt to read from a user-space location
75 : : * @dst: pointer to the buffer that shall take the data
76 : : * @src: address to read from. This must be a user address.
77 : : * @size: size of the data chunk
78 : : *
79 : : * Safely read from user address @src to the buffer at @dst. If a kernel fault
80 : : * happens, handle that and return -EFAULT.
81 : : */
82 : :
83 : : long __weak probe_user_read(void *dst, const void __user *src, size_t size)
84 : : __attribute__((alias("__probe_user_read")));
85 : :
86 : 0 : long __probe_user_read(void *dst, const void __user *src, size_t size)
87 : : {
88 : 0 : long ret = -EFAULT;
89 [ # # # ]: 0 : mm_segment_t old_fs = get_fs();
90 : :
91 [ # # # ]: 0 : set_fs(USER_DS);
92 [ # # # # ]: 0 : if (access_ok(src, size))
93 : 0 : ret = probe_read_common(dst, src, size);
94 : 0 : set_fs(old_fs);
95 : :
96 : 0 : return ret;
97 : : }
98 : : EXPORT_SYMBOL_GPL(probe_user_read);
99 : :
100 : : /**
101 : : * probe_kernel_write(): safely attempt to write to a location
102 : : * @dst: address to write to
103 : : * @src: pointer to the data that shall be written
104 : : * @size: size of the data chunk
105 : : *
106 : : * Safely write to address @dst from the buffer at @src. If a kernel fault
107 : : * happens, handle that and return -EFAULT.
108 : : */
109 : :
110 : : long __weak probe_kernel_write(void *dst, const void *src, size_t size)
111 : : __attribute__((alias("__probe_kernel_write")));
112 : :
113 : 0 : long __probe_kernel_write(void *dst, const void *src, size_t size)
114 : : {
115 : 0 : long ret;
116 : 0 : mm_segment_t old_fs = get_fs();
117 : :
118 : 0 : set_fs(KERNEL_DS);
119 : 0 : ret = probe_write_common((__force void __user *)dst, src, size);
120 : 0 : set_fs(old_fs);
121 : :
122 : 0 : return ret;
123 : : }
124 : : EXPORT_SYMBOL_GPL(probe_kernel_write);
125 : :
126 : : /**
127 : : * probe_user_write(): safely attempt to write to a user-space location
128 : : * @dst: address to write to
129 : : * @src: pointer to the data that shall be written
130 : : * @size: size of the data chunk
131 : : *
132 : : * Safely write to address @dst from the buffer at @src. If a kernel fault
133 : : * happens, handle that and return -EFAULT.
134 : : */
135 : :
136 : : long __weak probe_user_write(void __user *dst, const void *src, size_t size)
137 : : __attribute__((alias("__probe_user_write")));
138 : :
139 : 0 : long __probe_user_write(void __user *dst, const void *src, size_t size)
140 : : {
141 : 0 : long ret = -EFAULT;
142 [ # # # ]: 0 : mm_segment_t old_fs = get_fs();
143 : :
144 [ # # # ]: 0 : set_fs(USER_DS);
145 [ # # # # ]: 0 : if (access_ok(dst, size))
146 : 0 : ret = probe_write_common(dst, src, size);
147 : 0 : set_fs(old_fs);
148 : :
149 : 0 : return ret;
150 : : }
151 : : EXPORT_SYMBOL_GPL(probe_user_write);
152 : :
153 : : /**
154 : : * strncpy_from_unsafe: - Copy a NUL terminated string from unsafe address.
155 : : * @dst: Destination address, in kernel space. This buffer must be at
156 : : * least @count bytes long.
157 : : * @unsafe_addr: Unsafe address.
158 : : * @count: Maximum number of bytes to copy, including the trailing NUL.
159 : : *
160 : : * Copies a NUL-terminated string from unsafe address to kernel buffer.
161 : : *
162 : : * On success, returns the length of the string INCLUDING the trailing NUL.
163 : : *
164 : : * If access fails, returns -EFAULT (some data may have been copied
165 : : * and the trailing NUL added).
166 : : *
167 : : * If @count is smaller than the length of the string, copies @count-1 bytes,
168 : : * sets the last byte of @dst buffer to NUL and returns @count.
169 : : *
170 : : * strncpy_from_unsafe_strict() is the same as strncpy_from_unsafe() except
171 : : * for the case where architectures have non-overlapping user and kernel address
172 : : * ranges: strncpy_from_unsafe_strict() will additionally return -EFAULT for
173 : : * probing memory on a user address range where strncpy_from_unsafe_user() is
174 : : * supposed to be used instead.
175 : : */
176 : :
177 : : long __weak strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count)
178 : : __attribute__((alias("__strncpy_from_unsafe")));
179 : :
180 : : long __weak strncpy_from_unsafe_strict(char *dst, const void *unsafe_addr,
181 : : long count)
182 : : __attribute__((alias("__strncpy_from_unsafe")));
183 : :
184 : 0 : long __strncpy_from_unsafe(char *dst, const void *unsafe_addr, long count)
185 : : {
186 [ # # ]: 0 : mm_segment_t old_fs = get_fs();
187 : 0 : const void *src = unsafe_addr;
188 : 0 : long ret;
189 : :
190 [ # # ]: 0 : if (unlikely(count <= 0))
191 : : return 0;
192 : :
193 : 0 : set_fs(KERNEL_DS);
194 : 0 : pagefault_disable();
195 : :
196 : 0 : do {
197 : 0 : ret = __get_user(*dst++, (const char __user __force *)src++);
198 [ # # # # : 0 : } while (dst[-1] && ret == 0 && src - unsafe_addr < count);
# # ]
199 : :
200 : 0 : dst[-1] = '\0';
201 : 0 : pagefault_enable();
202 : 0 : set_fs(old_fs);
203 : :
204 [ # # ]: 0 : return ret ? -EFAULT : src - unsafe_addr;
205 : : }
206 : :
207 : : /**
208 : : * strncpy_from_unsafe_user: - Copy a NUL terminated string from unsafe user
209 : : * address.
210 : : * @dst: Destination address, in kernel space. This buffer must be at
211 : : * least @count bytes long.
212 : : * @unsafe_addr: Unsafe user address.
213 : : * @count: Maximum number of bytes to copy, including the trailing NUL.
214 : : *
215 : : * Copies a NUL-terminated string from unsafe user address to kernel buffer.
216 : : *
217 : : * On success, returns the length of the string INCLUDING the trailing NUL.
218 : : *
219 : : * If access fails, returns -EFAULT (some data may have been copied
220 : : * and the trailing NUL added).
221 : : *
222 : : * If @count is smaller than the length of the string, copies @count-1 bytes,
223 : : * sets the last byte of @dst buffer to NUL and returns @count.
224 : : */
225 : 0 : long strncpy_from_unsafe_user(char *dst, const void __user *unsafe_addr,
226 : : long count)
227 : : {
228 [ # # ]: 0 : mm_segment_t old_fs = get_fs();
229 : 0 : long ret;
230 : :
231 [ # # ]: 0 : if (unlikely(count <= 0))
232 : : return 0;
233 : :
234 [ # # # ]: 0 : set_fs(USER_DS);
235 : 0 : pagefault_disable();
236 : 0 : ret = strncpy_from_user(dst, unsafe_addr, count);
237 : 0 : pagefault_enable();
238 : 0 : set_fs(old_fs);
239 : :
240 [ # # ]: 0 : if (ret >= count) {
241 : 0 : ret = count;
242 : 0 : dst[ret - 1] = '\0';
243 [ # # ]: 0 : } else if (ret > 0) {
244 : 0 : ret++;
245 : : }
246 : :
247 : : return ret;
248 : : }
249 : :
250 : : /**
251 : : * strnlen_unsafe_user: - Get the size of a user string INCLUDING final NUL.
252 : : * @unsafe_addr: The string to measure.
253 : : * @count: Maximum count (including NUL)
254 : : *
255 : : * Get the size of a NUL-terminated string in user space without pagefault.
256 : : *
257 : : * Returns the size of the string INCLUDING the terminating NUL.
258 : : *
259 : : * If the string is too long, returns a number larger than @count. User
260 : : * has to check the return value against "> count".
261 : : * On exception (or invalid count), returns 0.
262 : : *
263 : : * Unlike strnlen_user, this can be used from IRQ handler etc. because
264 : : * it disables pagefaults.
265 : : */
266 : 0 : long strnlen_unsafe_user(const void __user *unsafe_addr, long count)
267 : : {
268 [ # # # ]: 0 : mm_segment_t old_fs = get_fs();
269 : 0 : int ret;
270 : :
271 [ # # # ]: 0 : set_fs(USER_DS);
272 : 0 : pagefault_disable();
273 : 0 : ret = strnlen_user(unsafe_addr, count);
274 : 0 : pagefault_enable();
275 : 0 : set_fs(old_fs);
276 : :
277 : 0 : return ret;
278 : : }
|
