Branch data Line data Source code
1 : : /*
2 : : * SPDX-License-Identifier: MIT
3 : : *
4 : : * Copyright © 2008 Intel Corporation
5 : : */
6 : :
7 : : #include <linux/string.h>
8 : : #include <linux/bitops.h>
9 : : #include <drm/i915_drm.h>
10 : :
11 : : #include "i915_drv.h"
12 : : #include "i915_gem.h"
13 : : #include "i915_gem_ioctls.h"
14 : : #include "i915_gem_mman.h"
15 : : #include "i915_gem_object.h"
16 : :
17 : : /**
18 : : * DOC: buffer object tiling
19 : : *
20 : : * i915_gem_set_tiling_ioctl() and i915_gem_get_tiling_ioctl() is the userspace
21 : : * interface to declare fence register requirements.
22 : : *
23 : : * In principle GEM doesn't care at all about the internal data layout of an
24 : : * object, and hence it also doesn't care about tiling or swizzling. There's two
25 : : * exceptions:
26 : : *
27 : : * - For X and Y tiling the hardware provides detilers for CPU access, so called
28 : : * fences. Since there's only a limited amount of them the kernel must manage
29 : : * these, and therefore userspace must tell the kernel the object tiling if it
30 : : * wants to use fences for detiling.
31 : : * - On gen3 and gen4 platforms have a swizzling pattern for tiled objects which
32 : : * depends upon the physical page frame number. When swapping such objects the
33 : : * page frame number might change and the kernel must be able to fix this up
34 : : * and hence now the tiling. Note that on a subset of platforms with
35 : : * asymmetric memory channel population the swizzling pattern changes in an
36 : : * unknown way, and for those the kernel simply forbids swapping completely.
37 : : *
38 : : * Since neither of this applies for new tiling layouts on modern platforms like
39 : : * W, Ys and Yf tiling GEM only allows object tiling to be set to X or Y tiled.
40 : : * Anything else can be handled in userspace entirely without the kernel's
41 : : * invovlement.
42 : : */
43 : :
44 : : /**
45 : : * i915_gem_fence_size - required global GTT size for a fence
46 : : * @i915: i915 device
47 : : * @size: object size
48 : : * @tiling: tiling mode
49 : : * @stride: tiling stride
50 : : *
51 : : * Return the required global GTT size for a fence (view of a tiled object),
52 : : * taking into account potential fence register mapping.
53 : : */
54 : 0 : u32 i915_gem_fence_size(struct drm_i915_private *i915,
55 : : u32 size, unsigned int tiling, unsigned int stride)
56 : : {
57 : 0 : u32 ggtt_size;
58 : :
59 : 0 : GEM_BUG_ON(!size);
60 : :
61 [ # # ]: 0 : if (tiling == I915_TILING_NONE)
62 : : return size;
63 : :
64 : 0 : GEM_BUG_ON(!stride);
65 : :
66 [ # # # # : 0 : if (INTEL_GEN(i915) >= 4) {
# # ]
67 [ # # # # : 0 : stride *= i915_gem_tile_height(tiling);
# # ]
68 : 0 : GEM_BUG_ON(!IS_ALIGNED(stride, I965_FENCE_PAGE));
69 : 0 : return roundup(size, stride);
70 : : }
71 : :
72 : : /* Previous chips need a power-of-two fence region when tiling */
73 [ # # # # : 0 : if (IS_GEN(i915, 3))
# # ]
74 : : ggtt_size = 1024*1024;
75 : : else
76 : 0 : ggtt_size = 512*1024;
77 : :
78 [ # # # # : 0 : while (ggtt_size < size)
# # # # #
# # # ]
79 : 0 : ggtt_size <<= 1;
80 : :
81 : : return ggtt_size;
82 : : }
83 : :
84 : : /**
85 : : * i915_gem_fence_alignment - required global GTT alignment for a fence
86 : : * @i915: i915 device
87 : : * @size: object size
88 : : * @tiling: tiling mode
89 : : * @stride: tiling stride
90 : : *
91 : : * Return the required global GTT alignment for a fence (a view of a tiled
92 : : * object), taking into account potential fence register mapping.
93 : : */
94 : 0 : u32 i915_gem_fence_alignment(struct drm_i915_private *i915, u32 size,
95 : : unsigned int tiling, unsigned int stride)
96 : : {
97 : 0 : GEM_BUG_ON(!size);
98 : :
99 : : /*
100 : : * Minimum alignment is 4k (GTT page size), but might be greater
101 : : * if a fence register is needed for the object.
102 : : */
103 [ # # ]: 0 : if (tiling == I915_TILING_NONE)
104 : : return I915_GTT_MIN_ALIGNMENT;
105 : :
106 [ # # # # : 0 : if (INTEL_GEN(i915) >= 4)
# # ]
107 : : return I965_FENCE_PAGE;
108 : :
109 : : /*
110 : : * Previous chips need to be aligned to the size of the smallest
111 : : * fence register that can contain the object.
112 : : */
113 [ # # # # : 0 : return i915_gem_fence_size(i915, size, tiling, stride);
# # ]
114 : : }
115 : :
116 : : /* Check pitch constriants for all chips & tiling formats */
117 : : static bool
118 : : i915_tiling_ok(struct drm_i915_gem_object *obj,
119 : : unsigned int tiling, unsigned int stride)
120 : : {
121 : : struct drm_i915_private *i915 = to_i915(obj->base.dev);
122 : : unsigned int tile_width;
123 : :
124 : : /* Linear is always fine */
125 : : if (tiling == I915_TILING_NONE)
126 : : return true;
127 : :
128 : : if (tiling > I915_TILING_LAST)
129 : : return false;
130 : :
131 : : /* check maximum stride & object size */
132 : : /* i965+ stores the end address of the gtt mapping in the fence
133 : : * reg, so dont bother to check the size */
134 : : if (INTEL_GEN(i915) >= 7) {
135 : : if (stride / 128 > GEN7_FENCE_MAX_PITCH_VAL)
136 : : return false;
137 : : } else if (INTEL_GEN(i915) >= 4) {
138 : : if (stride / 128 > I965_FENCE_MAX_PITCH_VAL)
139 : : return false;
140 : : } else {
141 : : if (stride > 8192)
142 : : return false;
143 : :
144 : : if (!is_power_of_2(stride))
145 : : return false;
146 : : }
147 : :
148 : : if (IS_GEN(i915, 2) ||
149 : : (tiling == I915_TILING_Y && HAS_128_BYTE_Y_TILING(i915)))
150 : : tile_width = 128;
151 : : else
152 : : tile_width = 512;
153 : :
154 : : if (!stride || !IS_ALIGNED(stride, tile_width))
155 : : return false;
156 : :
157 : : return true;
158 : : }
159 : :
160 : 0 : static bool i915_vma_fence_prepare(struct i915_vma *vma,
161 : : int tiling_mode, unsigned int stride)
162 : : {
163 : 0 : struct drm_i915_private *i915 = vma->vm->i915;
164 : 0 : u32 size, alignment;
165 : :
166 [ # # ]: 0 : if (!i915_vma_is_map_and_fenceable(vma))
167 : : return true;
168 : :
169 [ # # ]: 0 : size = i915_gem_fence_size(i915, vma->size, tiling_mode, stride);
170 [ # # ]: 0 : if (vma->node.size < size)
171 : : return false;
172 : :
173 [ # # ]: 0 : alignment = i915_gem_fence_alignment(i915, vma->size, tiling_mode, stride);
174 [ # # ]: 0 : if (!IS_ALIGNED(vma->node.start, alignment))
175 : 0 : return false;
176 : :
177 : : return true;
178 : : }
179 : :
180 : : /* Make the current GTT allocation valid for the change in tiling. */
181 : : static int
182 : 0 : i915_gem_object_fence_prepare(struct drm_i915_gem_object *obj,
183 : : int tiling_mode, unsigned int stride)
184 : : {
185 [ # # ]: 0 : struct i915_ggtt *ggtt = &to_i915(obj->base.dev)->ggtt;
186 : 0 : struct i915_vma *vma;
187 : 0 : int ret = 0;
188 : :
189 [ # # ]: 0 : if (tiling_mode == I915_TILING_NONE)
190 : : return 0;
191 : :
192 : 0 : mutex_lock(&ggtt->vm.mutex);
193 [ # # # # ]: 0 : for_each_ggtt_vma(vma, obj) {
194 [ # # ]: 0 : if (i915_vma_fence_prepare(vma, tiling_mode, stride))
195 : 0 : continue;
196 : :
197 : 0 : ret = __i915_vma_unbind(vma);
198 [ # # ]: 0 : if (ret)
199 : : break;
200 : : }
201 : 0 : mutex_unlock(&ggtt->vm.mutex);
202 : :
203 : 0 : return ret;
204 : : }
205 : :
206 : : int
207 : 0 : i915_gem_object_set_tiling(struct drm_i915_gem_object *obj,
208 : : unsigned int tiling, unsigned int stride)
209 : : {
210 [ # # ]: 0 : struct drm_i915_private *i915 = to_i915(obj->base.dev);
211 : 0 : struct i915_vma *vma;
212 : 0 : int err;
213 : :
214 : : /* Make sure we don't cross-contaminate obj->tiling_and_stride */
215 : 0 : BUILD_BUG_ON(I915_TILING_LAST & STRIDE_MASK);
216 : :
217 : 0 : GEM_BUG_ON(!i915_tiling_ok(obj, tiling, stride));
218 : 0 : GEM_BUG_ON(!stride ^ (tiling == I915_TILING_NONE));
219 : :
220 [ # # ]: 0 : if ((tiling | stride) == obj->tiling_and_stride)
221 : : return 0;
222 : :
223 [ # # ]: 0 : if (i915_gem_object_is_framebuffer(obj))
224 : : return -EBUSY;
225 : :
226 : : /* We need to rebind the object if its current allocation
227 : : * no longer meets the alignment restrictions for its new
228 : : * tiling mode. Otherwise we can just leave it alone, but
229 : : * need to ensure that any fence register is updated before
230 : : * the next fenced (either through the GTT or by the BLT unit
231 : : * on older GPUs) access.
232 : : *
233 : : * After updating the tiling parameters, we then flag whether
234 : : * we need to update an associated fence register. Note this
235 : : * has to also include the unfenced register the GPU uses
236 : : * whilst executing a fenced command for an untiled object.
237 : : */
238 : :
239 : 0 : i915_gem_object_lock(obj);
240 [ # # ]: 0 : if (i915_gem_object_is_framebuffer(obj)) {
241 : 0 : i915_gem_object_unlock(obj);
242 : 0 : return -EBUSY;
243 : : }
244 : :
245 : 0 : err = i915_gem_object_fence_prepare(obj, tiling, stride);
246 [ # # ]: 0 : if (err) {
247 : 0 : i915_gem_object_unlock(obj);
248 : 0 : return err;
249 : : }
250 : :
251 : : /* If the memory has unknown (i.e. varying) swizzling, we pin the
252 : : * pages to prevent them being swapped out and causing corruption
253 : : * due to the change in swizzling.
254 : : */
255 : 0 : mutex_lock(&obj->mm.lock);
256 [ # # # # ]: 0 : if (i915_gem_object_has_pages(obj) &&
257 [ # # ]: 0 : obj->mm.madv == I915_MADV_WILLNEED &&
258 [ # # ]: 0 : i915->quirks & QUIRK_PIN_SWIZZLED_PAGES) {
259 [ # # ]: 0 : if (tiling == I915_TILING_NONE) {
260 : 0 : GEM_BUG_ON(!obj->mm.quirked);
261 : 0 : __i915_gem_object_unpin_pages(obj);
262 : 0 : obj->mm.quirked = false;
263 : : }
264 [ # # ]: 0 : if (!i915_gem_object_is_tiled(obj)) {
265 : 0 : GEM_BUG_ON(obj->mm.quirked);
266 : 0 : __i915_gem_object_pin_pages(obj);
267 : 0 : obj->mm.quirked = true;
268 : : }
269 : : }
270 : 0 : mutex_unlock(&obj->mm.lock);
271 : :
272 [ # # # # ]: 0 : for_each_ggtt_vma(vma, obj) {
273 : 0 : vma->fence_size =
274 [ # # ]: 0 : i915_gem_fence_size(i915, vma->size, tiling, stride);
275 [ # # ]: 0 : vma->fence_alignment =
276 : : i915_gem_fence_alignment(i915,
277 : : vma->size, tiling, stride);
278 : :
279 [ # # ]: 0 : if (vma->fence)
280 : 0 : vma->fence->dirty = true;
281 : : }
282 : :
283 : 0 : obj->tiling_and_stride = tiling | stride;
284 : 0 : i915_gem_object_unlock(obj);
285 : :
286 : : /* Force the fence to be reacquired for GTT access */
287 : 0 : i915_gem_object_release_mmap(obj);
288 : :
289 : : /* Try to preallocate memory required to save swizzling on put-pages */
290 [ # # # # ]: 0 : if (i915_gem_object_needs_bit17_swizzle(obj)) {
291 [ # # ]: 0 : if (!obj->bit_17) {
292 : 0 : obj->bit_17 = bitmap_zalloc(obj->base.size >> PAGE_SHIFT,
293 : : GFP_KERNEL);
294 : : }
295 : : } else {
296 : 0 : bitmap_free(obj->bit_17);
297 : 0 : obj->bit_17 = NULL;
298 : : }
299 : :
300 : : return 0;
301 : : }
302 : :
303 : : /**
304 : : * i915_gem_set_tiling_ioctl - IOCTL handler to set tiling mode
305 : : * @dev: DRM device
306 : : * @data: data pointer for the ioctl
307 : : * @file: DRM file for the ioctl call
308 : : *
309 : : * Sets the tiling mode of an object, returning the required swizzling of
310 : : * bit 6 of addresses in the object.
311 : : *
312 : : * Called by the user via ioctl.
313 : : *
314 : : * Returns:
315 : : * Zero on success, negative errno on failure.
316 : : */
317 : : int
318 : 0 : i915_gem_set_tiling_ioctl(struct drm_device *dev, void *data,
319 : : struct drm_file *file)
320 : : {
321 [ # # ]: 0 : struct drm_i915_private *dev_priv = to_i915(dev);
322 : 0 : struct drm_i915_gem_set_tiling *args = data;
323 : 0 : struct drm_i915_gem_object *obj;
324 : 0 : int err;
325 : :
326 [ # # ]: 0 : if (!dev_priv->ggtt.num_fences)
327 : : return -EOPNOTSUPP;
328 : :
329 : 0 : obj = i915_gem_object_lookup(file, args->handle);
330 [ # # ]: 0 : if (!obj)
331 : : return -ENOENT;
332 : :
333 : : /*
334 : : * The tiling mode of proxy objects is handled by its generator, and
335 : : * not allowed to be changed by userspace.
336 : : */
337 [ # # ]: 0 : if (i915_gem_object_is_proxy(obj)) {
338 : 0 : err = -ENXIO;
339 : 0 : goto err;
340 : : }
341 : :
342 [ # # ]: 0 : if (!i915_tiling_ok(obj, args->tiling_mode, args->stride)) {
343 : 0 : err = -EINVAL;
344 : 0 : goto err;
345 : : }
346 : :
347 [ # # ]: 0 : if (args->tiling_mode == I915_TILING_NONE) {
348 : 0 : args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
349 : 0 : args->stride = 0;
350 : : } else {
351 [ # # ]: 0 : if (args->tiling_mode == I915_TILING_X)
352 : 0 : args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_x;
353 : : else
354 : 0 : args->swizzle_mode = to_i915(dev)->ggtt.bit_6_swizzle_y;
355 : :
356 : : /* Hide bit 17 swizzling from the user. This prevents old Mesa
357 : : * from aborting the application on sw fallbacks to bit 17,
358 : : * and we use the pread/pwrite bit17 paths to swizzle for it.
359 : : * If there was a user that was relying on the swizzle
360 : : * information for drm_intel_bo_map()ed reads/writes this would
361 : : * break it, but we don't have any of those.
362 : : */
363 [ # # ]: 0 : if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
364 : 0 : args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
365 [ # # ]: 0 : if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
366 : 0 : args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
367 : :
368 : : /* If we can't handle the swizzling, make it untiled. */
369 [ # # ]: 0 : if (args->swizzle_mode == I915_BIT_6_SWIZZLE_UNKNOWN) {
370 : 0 : args->tiling_mode = I915_TILING_NONE;
371 : 0 : args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
372 : 0 : args->stride = 0;
373 : : }
374 : : }
375 : :
376 : 0 : err = i915_gem_object_set_tiling(obj, args->tiling_mode, args->stride);
377 : :
378 : : /* We have to maintain this existing ABI... */
379 : 0 : args->stride = i915_gem_object_get_stride(obj);
380 : 0 : args->tiling_mode = i915_gem_object_get_tiling(obj);
381 : :
382 : 0 : err:
383 : 0 : i915_gem_object_put(obj);
384 : 0 : return err;
385 : : }
386 : :
387 : : /**
388 : : * i915_gem_get_tiling_ioctl - IOCTL handler to get tiling mode
389 : : * @dev: DRM device
390 : : * @data: data pointer for the ioctl
391 : : * @file: DRM file for the ioctl call
392 : : *
393 : : * Returns the current tiling mode and required bit 6 swizzling for the object.
394 : : *
395 : : * Called by the user via ioctl.
396 : : *
397 : : * Returns:
398 : : * Zero on success, negative errno on failure.
399 : : */
400 : : int
401 : 0 : i915_gem_get_tiling_ioctl(struct drm_device *dev, void *data,
402 : : struct drm_file *file)
403 : : {
404 : 0 : struct drm_i915_gem_get_tiling *args = data;
405 [ # # ]: 0 : struct drm_i915_private *dev_priv = to_i915(dev);
406 : 0 : struct drm_i915_gem_object *obj;
407 : 0 : int err = -ENOENT;
408 : :
409 [ # # ]: 0 : if (!dev_priv->ggtt.num_fences)
410 : : return -EOPNOTSUPP;
411 : :
412 : 0 : rcu_read_lock();
413 : 0 : obj = i915_gem_object_lookup_rcu(file, args->handle);
414 [ # # ]: 0 : if (obj) {
415 : 0 : args->tiling_mode =
416 : 0 : READ_ONCE(obj->tiling_and_stride) & TILING_MASK;
417 : 0 : err = 0;
418 : : }
419 : 0 : rcu_read_unlock();
420 [ # # ]: 0 : if (unlikely(err))
421 : : return err;
422 : :
423 [ # # # ]: 0 : switch (args->tiling_mode) {
424 : 0 : case I915_TILING_X:
425 : 0 : args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_x;
426 : 0 : break;
427 : 0 : case I915_TILING_Y:
428 : 0 : args->swizzle_mode = dev_priv->ggtt.bit_6_swizzle_y;
429 : 0 : break;
430 : 0 : default:
431 : : case I915_TILING_NONE:
432 : 0 : args->swizzle_mode = I915_BIT_6_SWIZZLE_NONE;
433 : 0 : break;
434 : : }
435 : :
436 : : /* Hide bit 17 from the user -- see comment in i915_gem_set_tiling */
437 [ # # ]: 0 : if (dev_priv->quirks & QUIRK_PIN_SWIZZLED_PAGES)
438 : 0 : args->phys_swizzle_mode = I915_BIT_6_SWIZZLE_UNKNOWN;
439 : : else
440 : 0 : args->phys_swizzle_mode = args->swizzle_mode;
441 [ # # ]: 0 : if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_17)
442 : 0 : args->swizzle_mode = I915_BIT_6_SWIZZLE_9;
443 [ # # ]: 0 : if (args->swizzle_mode == I915_BIT_6_SWIZZLE_9_10_17)
444 : 0 : args->swizzle_mode = I915_BIT_6_SWIZZLE_9_10;
445 : :
446 : : return 0;
447 : : }
|
