Branch data Line data Source code
1 : : /*
2 : : * SPDX-License-Identifier: MIT
3 : : *
4 : : * Copyright © 2014-2016 Intel Corporation
5 : : */
6 : :
7 : : #include "display/intel_frontbuffer.h"
8 : :
9 : : #include "i915_drv.h"
10 : : #include "i915_gem_clflush.h"
11 : : #include "i915_gem_gtt.h"
12 : : #include "i915_gem_ioctls.h"
13 : : #include "i915_gem_object.h"
14 : : #include "i915_vma.h"
15 : : #include "i915_gem_lmem.h"
16 : : #include "i915_gem_mman.h"
17 : :
18 : 0 : static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)
19 : : {
20 : : /*
21 : : * We manually flush the CPU domain so that we can override and
22 : : * force the flush for the display, and perform it asyncrhonously.
23 : : */
24 : 0 : i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
25 [ # # ]: 0 : if (obj->cache_dirty)
26 : 0 : i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);
27 : 0 : obj->write_domain = 0;
28 : 0 : }
29 : :
30 : 0 : void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)
31 : : {
32 [ # # ]: 0 : if (!i915_gem_object_is_framebuffer(obj))
33 : : return;
34 : :
35 : 0 : i915_gem_object_lock(obj);
36 : 0 : __i915_gem_object_flush_for_display(obj);
37 : 0 : i915_gem_object_unlock(obj);
38 : : }
39 : :
40 : : /**
41 : : * Moves a single object to the WC read, and possibly write domain.
42 : : * @obj: object to act on
43 : : * @write: ask for write access or read only
44 : : *
45 : : * This function returns when the move is complete, including waiting on
46 : : * flushes to occur.
47 : : */
48 : : int
49 : 0 : i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)
50 : : {
51 : 0 : int ret;
52 : :
53 : 0 : assert_object_held(obj);
54 : :
55 [ # # ]: 0 : ret = i915_gem_object_wait(obj,
56 : : I915_WAIT_INTERRUPTIBLE |
57 : : (write ? I915_WAIT_ALL : 0),
58 : : MAX_SCHEDULE_TIMEOUT);
59 [ # # ]: 0 : if (ret)
60 : : return ret;
61 : :
62 [ # # ]: 0 : if (obj->write_domain == I915_GEM_DOMAIN_WC)
63 : : return 0;
64 : :
65 : : /* Flush and acquire obj->pages so that we are coherent through
66 : : * direct access in memory with previous cached writes through
67 : : * shmemfs and that our cache domain tracking remains valid.
68 : : * For example, if the obj->filp was moved to swap without us
69 : : * being notified and releasing the pages, we would mistakenly
70 : : * continue to assume that the obj remained out of the CPU cached
71 : : * domain.
72 : : */
73 : 0 : ret = i915_gem_object_pin_pages(obj);
74 [ # # ]: 0 : if (ret)
75 : : return ret;
76 : :
77 : 0 : i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);
78 : :
79 : : /* Serialise direct access to this object with the barriers for
80 : : * coherent writes from the GPU, by effectively invalidating the
81 : : * WC domain upon first access.
82 : : */
83 [ # # ]: 0 : if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)
84 : 0 : mb();
85 : :
86 : : /* It should now be out of any other write domains, and we can update
87 : : * the domain values for our changes.
88 : : */
89 : 0 : GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);
90 : 0 : obj->read_domains |= I915_GEM_DOMAIN_WC;
91 [ # # ]: 0 : if (write) {
92 : 0 : obj->read_domains = I915_GEM_DOMAIN_WC;
93 : 0 : obj->write_domain = I915_GEM_DOMAIN_WC;
94 : 0 : obj->mm.dirty = true;
95 : : }
96 : :
97 : 0 : i915_gem_object_unpin_pages(obj);
98 : 0 : return 0;
99 : : }
100 : :
101 : : /**
102 : : * Moves a single object to the GTT read, and possibly write domain.
103 : : * @obj: object to act on
104 : : * @write: ask for write access or read only
105 : : *
106 : : * This function returns when the move is complete, including waiting on
107 : : * flushes to occur.
108 : : */
109 : : int
110 : 0 : i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)
111 : : {
112 : 0 : int ret;
113 : :
114 : 0 : assert_object_held(obj);
115 : :
116 [ # # ]: 0 : ret = i915_gem_object_wait(obj,
117 : : I915_WAIT_INTERRUPTIBLE |
118 : : (write ? I915_WAIT_ALL : 0),
119 : : MAX_SCHEDULE_TIMEOUT);
120 [ # # ]: 0 : if (ret)
121 : : return ret;
122 : :
123 [ # # ]: 0 : if (obj->write_domain == I915_GEM_DOMAIN_GTT)
124 : : return 0;
125 : :
126 : : /* Flush and acquire obj->pages so that we are coherent through
127 : : * direct access in memory with previous cached writes through
128 : : * shmemfs and that our cache domain tracking remains valid.
129 : : * For example, if the obj->filp was moved to swap without us
130 : : * being notified and releasing the pages, we would mistakenly
131 : : * continue to assume that the obj remained out of the CPU cached
132 : : * domain.
133 : : */
134 : 0 : ret = i915_gem_object_pin_pages(obj);
135 [ # # ]: 0 : if (ret)
136 : : return ret;
137 : :
138 : 0 : i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);
139 : :
140 : : /* Serialise direct access to this object with the barriers for
141 : : * coherent writes from the GPU, by effectively invalidating the
142 : : * GTT domain upon first access.
143 : : */
144 [ # # ]: 0 : if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)
145 : 0 : mb();
146 : :
147 : : /* It should now be out of any other write domains, and we can update
148 : : * the domain values for our changes.
149 : : */
150 : 0 : GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);
151 : 0 : obj->read_domains |= I915_GEM_DOMAIN_GTT;
152 [ # # ]: 0 : if (write) {
153 : 0 : struct i915_vma *vma;
154 : :
155 : 0 : obj->read_domains = I915_GEM_DOMAIN_GTT;
156 : 0 : obj->write_domain = I915_GEM_DOMAIN_GTT;
157 : 0 : obj->mm.dirty = true;
158 : :
159 : 0 : spin_lock(&obj->vma.lock);
160 [ # # # # ]: 0 : for_each_ggtt_vma(vma, obj)
161 [ # # ]: 0 : if (i915_vma_is_bound(vma, I915_VMA_GLOBAL_BIND))
162 : 0 : i915_vma_set_ggtt_write(vma);
163 : 0 : spin_unlock(&obj->vma.lock);
164 : : }
165 : :
166 : 0 : i915_gem_object_unpin_pages(obj);
167 : 0 : return 0;
168 : : }
169 : :
170 : : /**
171 : : * Changes the cache-level of an object across all VMA.
172 : : * @obj: object to act on
173 : : * @cache_level: new cache level to set for the object
174 : : *
175 : : * After this function returns, the object will be in the new cache-level
176 : : * across all GTT and the contents of the backing storage will be coherent,
177 : : * with respect to the new cache-level. In order to keep the backing storage
178 : : * coherent for all users, we only allow a single cache level to be set
179 : : * globally on the object and prevent it from being changed whilst the
180 : : * hardware is reading from the object. That is if the object is currently
181 : : * on the scanout it will be set to uncached (or equivalent display
182 : : * cache coherency) and all non-MOCS GPU access will also be uncached so
183 : : * that all direct access to the scanout remains coherent.
184 : : */
185 : 0 : int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,
186 : : enum i915_cache_level cache_level)
187 : : {
188 : 0 : int ret;
189 : :
190 [ # # ]: 0 : if (obj->cache_level == cache_level)
191 : : return 0;
192 : :
193 : 0 : ret = i915_gem_object_wait(obj,
194 : : I915_WAIT_INTERRUPTIBLE |
195 : : I915_WAIT_ALL,
196 : : MAX_SCHEDULE_TIMEOUT);
197 [ # # ]: 0 : if (ret)
198 : : return ret;
199 : :
200 : 0 : ret = i915_gem_object_lock_interruptible(obj);
201 [ # # ]: 0 : if (ret)
202 : : return ret;
203 : :
204 : : /* Always invalidate stale cachelines */
205 [ # # ]: 0 : if (obj->cache_level != cache_level) {
206 : 0 : i915_gem_object_set_cache_coherency(obj, cache_level);
207 : 0 : obj->cache_dirty = true;
208 : : }
209 : :
210 : 0 : i915_gem_object_unlock(obj);
211 : :
212 : : /* The cache-level will be applied when each vma is rebound. */
213 : 0 : return i915_gem_object_unbind(obj,
214 : : I915_GEM_OBJECT_UNBIND_ACTIVE |
215 : : I915_GEM_OBJECT_UNBIND_BARRIER);
216 : : }
217 : :
218 : 0 : int i915_gem_get_caching_ioctl(struct drm_device *dev, void *data,
219 : : struct drm_file *file)
220 : : {
221 : 0 : struct drm_i915_gem_caching *args = data;
222 : 0 : struct drm_i915_gem_object *obj;
223 : 0 : int err = 0;
224 : :
225 : 0 : rcu_read_lock();
226 : 0 : obj = i915_gem_object_lookup_rcu(file, args->handle);
227 [ # # ]: 0 : if (!obj) {
228 : 0 : err = -ENOENT;
229 : 0 : goto out;
230 : : }
231 : :
232 [ # # # ]: 0 : switch (obj->cache_level) {
233 : 0 : case I915_CACHE_LLC:
234 : : case I915_CACHE_L3_LLC:
235 : 0 : args->caching = I915_CACHING_CACHED;
236 : 0 : break;
237 : :
238 : 0 : case I915_CACHE_WT:
239 : 0 : args->caching = I915_CACHING_DISPLAY;
240 : 0 : break;
241 : :
242 : 0 : default:
243 : 0 : args->caching = I915_CACHING_NONE;
244 : 0 : break;
245 : : }
246 : 0 : out:
247 : 0 : rcu_read_unlock();
248 : 0 : return err;
249 : : }
250 : :
251 : 0 : int i915_gem_set_caching_ioctl(struct drm_device *dev, void *data,
252 : : struct drm_file *file)
253 : : {
254 [ # # # # ]: 0 : struct drm_i915_private *i915 = to_i915(dev);
255 : 0 : struct drm_i915_gem_caching *args = data;
256 : 0 : struct drm_i915_gem_object *obj;
257 : 0 : enum i915_cache_level level;
258 : 0 : int ret = 0;
259 : :
260 [ # # # # ]: 0 : switch (args->caching) {
261 : : case I915_CACHING_NONE:
262 : : level = I915_CACHE_NONE;
263 : : break;
264 : 0 : case I915_CACHING_CACHED:
265 : : /*
266 : : * Due to a HW issue on BXT A stepping, GPU stores via a
267 : : * snooped mapping may leave stale data in a corresponding CPU
268 : : * cacheline, whereas normally such cachelines would get
269 : : * invalidated.
270 : : */
271 [ # # ]: 0 : if (!HAS_LLC(i915) && !HAS_SNOOP(i915))
272 : : return -ENODEV;
273 : :
274 : : level = I915_CACHE_LLC;
275 : : break;
276 : : case I915_CACHING_DISPLAY:
277 [ # # # # : 0 : level = HAS_WT(i915) ? I915_CACHE_WT : I915_CACHE_NONE;
# # ]
278 : : break;
279 : : default:
280 : : return -EINVAL;
281 : : }
282 : :
283 : 0 : obj = i915_gem_object_lookup(file, args->handle);
284 [ # # ]: 0 : if (!obj)
285 : : return -ENOENT;
286 : :
287 : : /*
288 : : * The caching mode of proxy object is handled by its generator, and
289 : : * not allowed to be changed by userspace.
290 : : */
291 [ # # ]: 0 : if (i915_gem_object_is_proxy(obj)) {
292 : 0 : ret = -ENXIO;
293 : 0 : goto out;
294 : : }
295 : :
296 : 0 : ret = i915_gem_object_set_cache_level(obj, level);
297 : :
298 : 0 : out:
299 : 0 : i915_gem_object_put(obj);
300 : 0 : return ret;
301 : : }
302 : :
303 : : /*
304 : : * Prepare buffer for display plane (scanout, cursors, etc). Can be called from
305 : : * an uninterruptible phase (modesetting) and allows any flushes to be pipelined
306 : : * (for pageflips). We only flush the caches while preparing the buffer for
307 : : * display, the callers are responsible for frontbuffer flush.
308 : : */
309 : : struct i915_vma *
310 : 0 : i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,
311 : : u32 alignment,
312 : : const struct i915_ggtt_view *view,
313 : : unsigned int flags)
314 : : {
315 [ # # ]: 0 : struct drm_i915_private *i915 = to_i915(obj->base.dev);
316 : 0 : struct i915_vma *vma;
317 : 0 : int ret;
318 : :
319 : : /* Frame buffer must be in LMEM (no migration yet) */
320 [ # # # # ]: 0 : if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))
321 : : return ERR_PTR(-EINVAL);
322 : :
323 : : /*
324 : : * The display engine is not coherent with the LLC cache on gen6. As
325 : : * a result, we make sure that the pinning that is about to occur is
326 : : * done with uncached PTEs. This is lowest common denominator for all
327 : : * chipsets.
328 : : *
329 : : * However for gen6+, we could do better by using the GFDT bit instead
330 : : * of uncaching, which would allow us to flush all the LLC-cached data
331 : : * with that bit in the PTE to main memory with just one PIPE_CONTROL.
332 : : */
333 [ # # ]: 0 : ret = i915_gem_object_set_cache_level(obj,
334 [ # # # # ]: 0 : HAS_WT(i915) ?
335 : : I915_CACHE_WT : I915_CACHE_NONE);
336 [ # # ]: 0 : if (ret)
337 : 0 : return ERR_PTR(ret);
338 : :
339 : : /*
340 : : * As the user may map the buffer once pinned in the display plane
341 : : * (e.g. libkms for the bootup splash), we have to ensure that we
342 : : * always use map_and_fenceable for all scanout buffers. However,
343 : : * it may simply be too big to fit into mappable, in which case
344 : : * put it anyway and hope that userspace can cope (but always first
345 : : * try to preserve the existing ABI).
346 : : */
347 [ # # ]: 0 : vma = ERR_PTR(-ENOSPC);
348 [ # # # # ]: 0 : if ((flags & PIN_MAPPABLE) == 0 &&
349 [ # # ]: 0 : (!view || view->type == I915_GGTT_VIEW_NORMAL))
350 : 0 : vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment,
351 : : flags |
352 : : PIN_MAPPABLE |
353 : : PIN_NONBLOCK);
354 [ # # ]: 0 : if (IS_ERR(vma))
355 : 0 : vma = i915_gem_object_ggtt_pin(obj, view, 0, alignment, flags);
356 [ # # ]: 0 : if (IS_ERR(vma))
357 : : return vma;
358 : :
359 : 0 : vma->display_alignment = max_t(u64, vma->display_alignment, alignment);
360 : :
361 : 0 : i915_gem_object_flush_if_display(obj);
362 : :
363 : 0 : return vma;
364 : : }
365 : :
366 : 0 : static void i915_gem_object_bump_inactive_ggtt(struct drm_i915_gem_object *obj)
367 : : {
368 : 0 : struct drm_i915_private *i915 = to_i915(obj->base.dev);
369 : 0 : struct i915_vma *vma;
370 : :
371 : 0 : GEM_BUG_ON(!i915_gem_object_has_pinned_pages(obj));
372 [ # # ]: 0 : if (!atomic_read(&obj->bind_count))
373 : : return;
374 : :
375 : 0 : mutex_lock(&i915->ggtt.vm.mutex);
376 : 0 : spin_lock(&obj->vma.lock);
377 [ # # # # ]: 0 : for_each_ggtt_vma(vma, obj) {
378 [ # # ]: 0 : if (!drm_mm_node_allocated(&vma->node))
379 : 0 : continue;
380 : :
381 : 0 : GEM_BUG_ON(vma->vm != &i915->ggtt.vm);
382 : 0 : list_move_tail(&vma->vm_link, &vma->vm->bound_list);
383 : : }
384 : 0 : spin_unlock(&obj->vma.lock);
385 : 0 : mutex_unlock(&i915->ggtt.vm.mutex);
386 : :
387 [ # # ]: 0 : if (i915_gem_object_is_shrinkable(obj)) {
388 : 0 : unsigned long flags;
389 : :
390 : 0 : spin_lock_irqsave(&i915->mm.obj_lock, flags);
391 : :
392 [ # # # # ]: 0 : if (obj->mm.madv == I915_MADV_WILLNEED &&
393 : 0 : !atomic_read(&obj->mm.shrink_pin))
394 : 0 : list_move_tail(&obj->mm.link, &i915->mm.shrink_list);
395 : :
396 : 0 : spin_unlock_irqrestore(&i915->mm.obj_lock, flags);
397 : : }
398 : : }
399 : :
400 : : void
401 : 0 : i915_gem_object_unpin_from_display_plane(struct i915_vma *vma)
402 : : {
403 : 0 : struct drm_i915_gem_object *obj = vma->obj;
404 : :
405 : 0 : assert_object_held(obj);
406 : :
407 : : /* Bump the LRU to try and avoid premature eviction whilst flipping */
408 : 0 : i915_gem_object_bump_inactive_ggtt(obj);
409 : :
410 : 0 : i915_vma_unpin(vma);
411 : 0 : }
412 : :
413 : : /**
414 : : * Moves a single object to the CPU read, and possibly write domain.
415 : : * @obj: object to act on
416 : : * @write: requesting write or read-only access
417 : : *
418 : : * This function returns when the move is complete, including waiting on
419 : : * flushes to occur.
420 : : */
421 : : int
422 : 0 : i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)
423 : : {
424 : 0 : int ret;
425 : :
426 : 0 : assert_object_held(obj);
427 : :
428 [ # # ]: 0 : ret = i915_gem_object_wait(obj,
429 : : I915_WAIT_INTERRUPTIBLE |
430 : : (write ? I915_WAIT_ALL : 0),
431 : : MAX_SCHEDULE_TIMEOUT);
432 [ # # ]: 0 : if (ret)
433 : : return ret;
434 : :
435 : 0 : i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
436 : :
437 : : /* Flush the CPU cache if it's still invalid. */
438 [ # # ]: 0 : if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {
439 : 0 : i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);
440 : 0 : obj->read_domains |= I915_GEM_DOMAIN_CPU;
441 : : }
442 : :
443 : : /* It should now be out of any other write domains, and we can update
444 : : * the domain values for our changes.
445 : : */
446 : 0 : GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);
447 : :
448 : : /* If we're writing through the CPU, then the GPU read domains will
449 : : * need to be invalidated at next use.
450 : : */
451 [ # # ]: 0 : if (write)
452 [ # # ]: 0 : __start_cpu_write(obj);
453 : :
454 : : return 0;
455 : : }
456 : :
457 : : /**
458 : : * Called when user space prepares to use an object with the CPU, either
459 : : * through the mmap ioctl's mapping or a GTT mapping.
460 : : * @dev: drm device
461 : : * @data: ioctl data blob
462 : : * @file: drm file
463 : : */
464 : : int
465 : 0 : i915_gem_set_domain_ioctl(struct drm_device *dev, void *data,
466 : : struct drm_file *file)
467 : : {
468 : 0 : struct drm_i915_gem_set_domain *args = data;
469 : 0 : struct drm_i915_gem_object *obj;
470 : 0 : u32 read_domains = args->read_domains;
471 : 0 : u32 write_domain = args->write_domain;
472 : 0 : int err;
473 : :
474 : : /* Only handle setting domains to types used by the CPU. */
475 [ # # ]: 0 : if ((write_domain | read_domains) & I915_GEM_GPU_DOMAINS)
476 : : return -EINVAL;
477 : :
478 : : /*
479 : : * Having something in the write domain implies it's in the read
480 : : * domain, and only that read domain. Enforce that in the request.
481 : : */
482 [ # # ]: 0 : if (write_domain && read_domains != write_domain)
483 : : return -EINVAL;
484 : :
485 [ # # ]: 0 : if (!read_domains)
486 : : return 0;
487 : :
488 : 0 : obj = i915_gem_object_lookup(file, args->handle);
489 [ # # ]: 0 : if (!obj)
490 : : return -ENOENT;
491 : :
492 : : /*
493 : : * Already in the desired write domain? Nothing for us to do!
494 : : *
495 : : * We apply a little bit of cunning here to catch a broader set of
496 : : * no-ops. If obj->write_domain is set, we must be in the same
497 : : * obj->read_domains, and only that domain. Therefore, if that
498 : : * obj->write_domain matches the request read_domains, we are
499 : : * already in the same read/write domain and can skip the operation,
500 : : * without having to further check the requested write_domain.
501 : : */
502 [ # # ]: 0 : if (READ_ONCE(obj->write_domain) == read_domains) {
503 : 0 : err = 0;
504 : 0 : goto out;
505 : : }
506 : :
507 : : /*
508 : : * Try to flush the object off the GPU without holding the lock.
509 : : * We will repeat the flush holding the lock in the normal manner
510 : : * to catch cases where we are gazumped.
511 : : */
512 [ # # ]: 0 : err = i915_gem_object_wait(obj,
513 : : I915_WAIT_INTERRUPTIBLE |
514 : : I915_WAIT_PRIORITY |
515 : : (write_domain ? I915_WAIT_ALL : 0),
516 : : MAX_SCHEDULE_TIMEOUT);
517 [ # # ]: 0 : if (err)
518 : 0 : goto out;
519 : :
520 : : /*
521 : : * Proxy objects do not control access to the backing storage, ergo
522 : : * they cannot be used as a means to manipulate the cache domain
523 : : * tracking for that backing storage. The proxy object is always
524 : : * considered to be outside of any cache domain.
525 : : */
526 [ # # ]: 0 : if (i915_gem_object_is_proxy(obj)) {
527 : 0 : err = -ENXIO;
528 : 0 : goto out;
529 : : }
530 : :
531 : : /*
532 : : * Flush and acquire obj->pages so that we are coherent through
533 : : * direct access in memory with previous cached writes through
534 : : * shmemfs and that our cache domain tracking remains valid.
535 : : * For example, if the obj->filp was moved to swap without us
536 : : * being notified and releasing the pages, we would mistakenly
537 : : * continue to assume that the obj remained out of the CPU cached
538 : : * domain.
539 : : */
540 : 0 : err = i915_gem_object_pin_pages(obj);
541 [ # # ]: 0 : if (err)
542 : 0 : goto out;
543 : :
544 : 0 : err = i915_gem_object_lock_interruptible(obj);
545 [ # # ]: 0 : if (err)
546 : 0 : goto out_unpin;
547 : :
548 [ # # ]: 0 : if (read_domains & I915_GEM_DOMAIN_WC)
549 : 0 : err = i915_gem_object_set_to_wc_domain(obj, write_domain);
550 [ # # ]: 0 : else if (read_domains & I915_GEM_DOMAIN_GTT)
551 : 0 : err = i915_gem_object_set_to_gtt_domain(obj, write_domain);
552 : : else
553 : 0 : err = i915_gem_object_set_to_cpu_domain(obj, write_domain);
554 : :
555 : : /* And bump the LRU for this access */
556 : 0 : i915_gem_object_bump_inactive_ggtt(obj);
557 : :
558 : 0 : i915_gem_object_unlock(obj);
559 : :
560 [ # # ]: 0 : if (write_domain)
561 [ # # ]: 0 : i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
562 : :
563 : 0 : out_unpin:
564 : 0 : i915_gem_object_unpin_pages(obj);
565 : 0 : out:
566 : 0 : i915_gem_object_put(obj);
567 : 0 : return err;
568 : : }
569 : :
570 : : /*
571 : : * Pins the specified object's pages and synchronizes the object with
572 : : * GPU accesses. Sets needs_clflush to non-zero if the caller should
573 : : * flush the object from the CPU cache.
574 : : */
575 : 0 : int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,
576 : : unsigned int *needs_clflush)
577 : : {
578 : 0 : int ret;
579 : :
580 : 0 : *needs_clflush = 0;
581 [ # # ]: 0 : if (!i915_gem_object_has_struct_page(obj))
582 : : return -ENODEV;
583 : :
584 : 0 : ret = i915_gem_object_lock_interruptible(obj);
585 [ # # ]: 0 : if (ret)
586 : : return ret;
587 : :
588 : 0 : ret = i915_gem_object_wait(obj,
589 : : I915_WAIT_INTERRUPTIBLE,
590 : : MAX_SCHEDULE_TIMEOUT);
591 [ # # ]: 0 : if (ret)
592 : 0 : goto err_unlock;
593 : :
594 : 0 : ret = i915_gem_object_pin_pages(obj);
595 [ # # ]: 0 : if (ret)
596 : 0 : goto err_unlock;
597 : :
598 [ # # ]: 0 : if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||
599 : : !static_cpu_has(X86_FEATURE_CLFLUSH)) {
600 : 0 : ret = i915_gem_object_set_to_cpu_domain(obj, false);
601 [ # # ]: 0 : if (ret)
602 : 0 : goto err_unpin;
603 : : else
604 : 0 : goto out;
605 : : }
606 : :
607 : 0 : i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
608 : :
609 : : /* If we're not in the cpu read domain, set ourself into the gtt
610 : : * read domain and manually flush cachelines (if required). This
611 : : * optimizes for the case when the gpu will dirty the data
612 : : * anyway again before the next pread happens.
613 : : */
614 [ # # ]: 0 : if (!obj->cache_dirty &&
615 [ # # ]: 0 : !(obj->read_domains & I915_GEM_DOMAIN_CPU))
616 : 0 : *needs_clflush = CLFLUSH_BEFORE;
617 : :
618 : 0 : out:
619 : : /* return with the pages pinned */
620 : : return 0;
621 : :
622 : : err_unpin:
623 : 0 : i915_gem_object_unpin_pages(obj);
624 : 0 : err_unlock:
625 : 0 : i915_gem_object_unlock(obj);
626 : 0 : return ret;
627 : : }
628 : :
629 : 0 : int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,
630 : : unsigned int *needs_clflush)
631 : : {
632 : 0 : int ret;
633 : :
634 : 0 : *needs_clflush = 0;
635 [ # # ]: 0 : if (!i915_gem_object_has_struct_page(obj))
636 : : return -ENODEV;
637 : :
638 : 0 : ret = i915_gem_object_lock_interruptible(obj);
639 [ # # ]: 0 : if (ret)
640 : : return ret;
641 : :
642 : 0 : ret = i915_gem_object_wait(obj,
643 : : I915_WAIT_INTERRUPTIBLE |
644 : : I915_WAIT_ALL,
645 : : MAX_SCHEDULE_TIMEOUT);
646 [ # # ]: 0 : if (ret)
647 : 0 : goto err_unlock;
648 : :
649 : 0 : ret = i915_gem_object_pin_pages(obj);
650 [ # # ]: 0 : if (ret)
651 : 0 : goto err_unlock;
652 : :
653 [ # # ]: 0 : if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||
654 : : !static_cpu_has(X86_FEATURE_CLFLUSH)) {
655 : 0 : ret = i915_gem_object_set_to_cpu_domain(obj, true);
656 [ # # ]: 0 : if (ret)
657 : 0 : goto err_unpin;
658 : : else
659 : 0 : goto out;
660 : : }
661 : :
662 : 0 : i915_gem_object_flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);
663 : :
664 : : /* If we're not in the cpu write domain, set ourself into the
665 : : * gtt write domain and manually flush cachelines (as required).
666 : : * This optimizes for the case when the gpu will use the data
667 : : * right away and we therefore have to clflush anyway.
668 : : */
669 [ # # ]: 0 : if (!obj->cache_dirty) {
670 : 0 : *needs_clflush |= CLFLUSH_AFTER;
671 : :
672 : : /*
673 : : * Same trick applies to invalidate partially written
674 : : * cachelines read before writing.
675 : : */
676 [ # # ]: 0 : if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))
677 : 0 : *needs_clflush |= CLFLUSH_BEFORE;
678 : : }
679 : :
680 : 0 : out:
681 [ # # ]: 0 : i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);
682 : 0 : obj->mm.dirty = true;
683 : : /* return with the pages pinned */
684 : 0 : return 0;
685 : :
686 : : err_unpin:
687 : 0 : i915_gem_object_unpin_pages(obj);
688 : 0 : err_unlock:
689 : 0 : i915_gem_object_unlock(obj);
690 : 0 : return ret;
691 : : }
|
