Branch data Line data Source code
1 : : // SPDX-License-Identifier: MIT
2 : : /*
3 : : * Copyright © 2010 Daniel Vetter
4 : : * Copyright © 2020 Intel Corporation
5 : : */
6 : :
7 : : #include <linux/slab.h> /* fault-inject.h is not standalone! */
8 : :
9 : : #include <linux/fault-inject.h>
10 : : #include <linux/log2.h>
11 : : #include <linux/random.h>
12 : : #include <linux/seq_file.h>
13 : : #include <linux/stop_machine.h>
14 : :
15 : : #include <asm/set_memory.h>
16 : : #include <asm/smp.h>
17 : :
18 : : #include <drm/i915_drm.h>
19 : :
20 : : #include "display/intel_frontbuffer.h"
21 : : #include "gt/intel_gt.h"
22 : : #include "gt/intel_gt_requests.h"
23 : :
24 : : #include "i915_drv.h"
25 : : #include "i915_scatterlist.h"
26 : : #include "i915_trace.h"
27 : : #include "i915_vgpu.h"
28 : :
29 : 0 : int i915_gem_gtt_prepare_pages(struct drm_i915_gem_object *obj,
30 : : struct sg_table *pages)
31 : : {
32 : 0 : do {
33 [ # # ]: 0 : if (dma_map_sg_attrs(&obj->base.dev->pdev->dev,
34 : 0 : pages->sgl, pages->nents,
35 : : PCI_DMA_BIDIRECTIONAL,
36 : : DMA_ATTR_NO_WARN))
37 : : return 0;
38 : :
39 : : /*
40 : : * If the DMA remap fails, one cause can be that we have
41 : : * too many objects pinned in a small remapping table,
42 : : * such as swiotlb. Incrementally purge all other objects and
43 : : * try again - if there are no more pages to remove from
44 : : * the DMA remapper, i915_gem_shrink will return 0.
45 : : */
46 : 0 : GEM_BUG_ON(obj->mm.pages == pages);
47 : 0 : } while (i915_gem_shrink(to_i915(obj->base.dev),
48 : 0 : obj->base.size >> PAGE_SHIFT, NULL,
49 : : I915_SHRINK_BOUND |
50 [ # # ]: 0 : I915_SHRINK_UNBOUND));
51 : :
52 : : return -ENOSPC;
53 : : }
54 : :
55 : 0 : void i915_gem_gtt_finish_pages(struct drm_i915_gem_object *obj,
56 : : struct sg_table *pages)
57 : : {
58 [ # # ]: 0 : struct drm_i915_private *dev_priv = to_i915(obj->base.dev);
59 : 0 : struct device *kdev = &dev_priv->drm.pdev->dev;
60 : 0 : struct i915_ggtt *ggtt = &dev_priv->ggtt;
61 : :
62 [ # # ]: 0 : if (unlikely(ggtt->do_idle_maps)) {
63 : : /* XXX This does not prevent more requests being submitted! */
64 [ # # ]: 0 : if (intel_gt_retire_requests_timeout(ggtt->vm.gt,
65 : : -MAX_SCHEDULE_TIMEOUT)) {
66 : 0 : DRM_ERROR("Failed to wait for idle; VT'd may hang.\n");
67 : : /* Wait a bit, in hopes it avoids the hang */
68 : 0 : udelay(10);
69 : : }
70 : : }
71 : :
72 : 0 : dma_unmap_sg(kdev, pages->sgl, pages->nents, PCI_DMA_BIDIRECTIONAL);
73 : 0 : }
74 : :
75 : : /**
76 : : * i915_gem_gtt_reserve - reserve a node in an address_space (GTT)
77 : : * @vm: the &struct i915_address_space
78 : : * @node: the &struct drm_mm_node (typically i915_vma.mode)
79 : : * @size: how much space to allocate inside the GTT,
80 : : * must be #I915_GTT_PAGE_SIZE aligned
81 : : * @offset: where to insert inside the GTT,
82 : : * must be #I915_GTT_MIN_ALIGNMENT aligned, and the node
83 : : * (@offset + @size) must fit within the address space
84 : : * @color: color to apply to node, if this node is not from a VMA,
85 : : * color must be #I915_COLOR_UNEVICTABLE
86 : : * @flags: control search and eviction behaviour
87 : : *
88 : : * i915_gem_gtt_reserve() tries to insert the @node at the exact @offset inside
89 : : * the address space (using @size and @color). If the @node does not fit, it
90 : : * tries to evict any overlapping nodes from the GTT, including any
91 : : * neighbouring nodes if the colors do not match (to ensure guard pages between
92 : : * differing domains). See i915_gem_evict_for_node() for the gory details
93 : : * on the eviction algorithm. #PIN_NONBLOCK may used to prevent waiting on
94 : : * evicting active overlapping objects, and any overlapping node that is pinned
95 : : * or marked as unevictable will also result in failure.
96 : : *
97 : : * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
98 : : * asked to wait for eviction and interrupted.
99 : : */
100 : 0 : int i915_gem_gtt_reserve(struct i915_address_space *vm,
101 : : struct drm_mm_node *node,
102 : : u64 size, u64 offset, unsigned long color,
103 : : unsigned int flags)
104 : : {
105 : 0 : int err;
106 : :
107 : 0 : GEM_BUG_ON(!size);
108 : 0 : GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
109 : 0 : GEM_BUG_ON(!IS_ALIGNED(offset, I915_GTT_MIN_ALIGNMENT));
110 : 0 : GEM_BUG_ON(range_overflows(offset, size, vm->total));
111 : 0 : GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm);
112 : 0 : GEM_BUG_ON(drm_mm_node_allocated(node));
113 : :
114 : 0 : node->size = size;
115 : 0 : node->start = offset;
116 : 0 : node->color = color;
117 : :
118 : 0 : err = drm_mm_reserve_node(&vm->mm, node);
119 [ # # ]: 0 : if (err != -ENOSPC)
120 : : return err;
121 : :
122 [ # # ]: 0 : if (flags & PIN_NOEVICT)
123 : : return -ENOSPC;
124 : :
125 : 0 : err = i915_gem_evict_for_node(vm, node, flags);
126 [ # # ]: 0 : if (err == 0)
127 : 0 : err = drm_mm_reserve_node(&vm->mm, node);
128 : :
129 : : return err;
130 : : }
131 : :
132 : 0 : static u64 random_offset(u64 start, u64 end, u64 len, u64 align)
133 : : {
134 : 0 : u64 range, addr;
135 : :
136 : 0 : GEM_BUG_ON(range_overflows(start, len, end));
137 : 0 : GEM_BUG_ON(round_up(start, align) > round_down(end - len, align));
138 : :
139 : 0 : range = round_down(end - len, align) - round_up(start, align);
140 [ # # ]: 0 : if (range) {
141 : 0 : if (sizeof(unsigned long) == sizeof(u64)) {
142 : 0 : addr = get_random_long();
143 : : } else {
144 : : addr = get_random_int();
145 : : if (range > U32_MAX) {
146 : : addr <<= 32;
147 : : addr |= get_random_int();
148 : : }
149 : : }
150 : 0 : div64_u64_rem(addr, range, &addr);
151 : 0 : start += addr;
152 : : }
153 : :
154 : 0 : return round_up(start, align);
155 : : }
156 : :
157 : : /**
158 : : * i915_gem_gtt_insert - insert a node into an address_space (GTT)
159 : : * @vm: the &struct i915_address_space
160 : : * @node: the &struct drm_mm_node (typically i915_vma.node)
161 : : * @size: how much space to allocate inside the GTT,
162 : : * must be #I915_GTT_PAGE_SIZE aligned
163 : : * @alignment: required alignment of starting offset, may be 0 but
164 : : * if specified, this must be a power-of-two and at least
165 : : * #I915_GTT_MIN_ALIGNMENT
166 : : * @color: color to apply to node
167 : : * @start: start of any range restriction inside GTT (0 for all),
168 : : * must be #I915_GTT_PAGE_SIZE aligned
169 : : * @end: end of any range restriction inside GTT (U64_MAX for all),
170 : : * must be #I915_GTT_PAGE_SIZE aligned if not U64_MAX
171 : : * @flags: control search and eviction behaviour
172 : : *
173 : : * i915_gem_gtt_insert() first searches for an available hole into which
174 : : * is can insert the node. The hole address is aligned to @alignment and
175 : : * its @size must then fit entirely within the [@start, @end] bounds. The
176 : : * nodes on either side of the hole must match @color, or else a guard page
177 : : * will be inserted between the two nodes (or the node evicted). If no
178 : : * suitable hole is found, first a victim is randomly selected and tested
179 : : * for eviction, otherwise then the LRU list of objects within the GTT
180 : : * is scanned to find the first set of replacement nodes to create the hole.
181 : : * Those old overlapping nodes are evicted from the GTT (and so must be
182 : : * rebound before any future use). Any node that is currently pinned cannot
183 : : * be evicted (see i915_vma_pin()). Similar if the node's VMA is currently
184 : : * active and #PIN_NONBLOCK is specified, that node is also skipped when
185 : : * searching for an eviction candidate. See i915_gem_evict_something() for
186 : : * the gory details on the eviction algorithm.
187 : : *
188 : : * Returns: 0 on success, -ENOSPC if no suitable hole is found, -EINTR if
189 : : * asked to wait for eviction and interrupted.
190 : : */
191 : 0 : int i915_gem_gtt_insert(struct i915_address_space *vm,
192 : : struct drm_mm_node *node,
193 : : u64 size, u64 alignment, unsigned long color,
194 : : u64 start, u64 end, unsigned int flags)
195 : : {
196 : 0 : enum drm_mm_insert_mode mode;
197 : 0 : u64 offset;
198 : 0 : int err;
199 : :
200 : 0 : lockdep_assert_held(&vm->mutex);
201 : :
202 : 0 : GEM_BUG_ON(!size);
203 : 0 : GEM_BUG_ON(!IS_ALIGNED(size, I915_GTT_PAGE_SIZE));
204 : 0 : GEM_BUG_ON(alignment && !is_power_of_2(alignment));
205 : 0 : GEM_BUG_ON(alignment && !IS_ALIGNED(alignment, I915_GTT_MIN_ALIGNMENT));
206 : 0 : GEM_BUG_ON(start >= end);
207 : 0 : GEM_BUG_ON(start > 0 && !IS_ALIGNED(start, I915_GTT_PAGE_SIZE));
208 : 0 : GEM_BUG_ON(end < U64_MAX && !IS_ALIGNED(end, I915_GTT_PAGE_SIZE));
209 : 0 : GEM_BUG_ON(vm == &vm->i915->ggtt.alias->vm);
210 : 0 : GEM_BUG_ON(drm_mm_node_allocated(node));
211 : :
212 [ # # # # : 0 : if (unlikely(range_overflows(start, size, end)))
# # ]
213 : : return -ENOSPC;
214 : :
215 [ # # ]: 0 : if (unlikely(round_up(start, alignment) > round_down(end - size, alignment)))
216 : : return -ENOSPC;
217 : :
218 : 0 : mode = DRM_MM_INSERT_BEST;
219 [ # # ]: 0 : if (flags & PIN_HIGH)
220 : 0 : mode = DRM_MM_INSERT_HIGHEST;
221 [ # # ]: 0 : if (flags & PIN_MAPPABLE)
222 : 0 : mode = DRM_MM_INSERT_LOW;
223 : :
224 : : /* We only allocate in PAGE_SIZE/GTT_PAGE_SIZE (4096) chunks,
225 : : * so we know that we always have a minimum alignment of 4096.
226 : : * The drm_mm range manager is optimised to return results
227 : : * with zero alignment, so where possible use the optimal
228 : : * path.
229 : : */
230 : 0 : BUILD_BUG_ON(I915_GTT_MIN_ALIGNMENT > I915_GTT_PAGE_SIZE);
231 [ # # ]: 0 : if (alignment <= I915_GTT_MIN_ALIGNMENT)
232 : 0 : alignment = 0;
233 : :
234 : 0 : err = drm_mm_insert_node_in_range(&vm->mm, node,
235 : : size, alignment, color,
236 : : start, end, mode);
237 [ # # ]: 0 : if (err != -ENOSPC)
238 : : return err;
239 : :
240 [ # # ]: 0 : if (mode & DRM_MM_INSERT_ONCE) {
241 : 0 : err = drm_mm_insert_node_in_range(&vm->mm, node,
242 : : size, alignment, color,
243 : : start, end,
244 : : DRM_MM_INSERT_BEST);
245 [ # # ]: 0 : if (err != -ENOSPC)
246 : : return err;
247 : : }
248 : :
249 [ # # ]: 0 : if (flags & PIN_NOEVICT)
250 : : return -ENOSPC;
251 : :
252 : : /*
253 : : * No free space, pick a slot at random.
254 : : *
255 : : * There is a pathological case here using a GTT shared between
256 : : * mmap and GPU (i.e. ggtt/aliasing_ppgtt but not full-ppgtt):
257 : : *
258 : : * |<-- 256 MiB aperture -->||<-- 1792 MiB unmappable -->|
259 : : * (64k objects) (448k objects)
260 : : *
261 : : * Now imagine that the eviction LRU is ordered top-down (just because
262 : : * pathology meets real life), and that we need to evict an object to
263 : : * make room inside the aperture. The eviction scan then has to walk
264 : : * the 448k list before it finds one within range. And now imagine that
265 : : * it has to search for a new hole between every byte inside the memcpy,
266 : : * for several simultaneous clients.
267 : : *
268 : : * On a full-ppgtt system, if we have run out of available space, there
269 : : * will be lots and lots of objects in the eviction list! Again,
270 : : * searching that LRU list may be slow if we are also applying any
271 : : * range restrictions (e.g. restriction to low 4GiB) and so, for
272 : : * simplicity and similarilty between different GTT, try the single
273 : : * random replacement first.
274 : : */
275 [ # # ]: 0 : offset = random_offset(start, end,
276 : : size, alignment ?: I915_GTT_MIN_ALIGNMENT);
277 : 0 : err = i915_gem_gtt_reserve(vm, node, size, offset, color, flags);
278 [ # # ]: 0 : if (err != -ENOSPC)
279 : : return err;
280 : :
281 [ # # ]: 0 : if (flags & PIN_NOSEARCH)
282 : : return -ENOSPC;
283 : :
284 : : /* Randomly selected placement is pinned, do a search */
285 : 0 : err = i915_gem_evict_something(vm, size, alignment, color,
286 : : start, end, flags);
287 [ # # ]: 0 : if (err)
288 : : return err;
289 : :
290 : 0 : return drm_mm_insert_node_in_range(&vm->mm, node,
291 : : size, alignment, color,
292 : : start, end, DRM_MM_INSERT_EVICT);
293 : : }
294 : :
295 : : #if IS_ENABLED(CONFIG_DRM_I915_SELFTEST)
296 : : #include "selftests/i915_gem_gtt.c"
297 : : #endif
|
