Branch data Line data Source code
1 : : /*
2 : : * drm_irq.c IRQ and vblank support
3 : : *
4 : : * \author Rickard E. (Rik) Faith <faith@valinux.com>
5 : : * \author Gareth Hughes <gareth@valinux.com>
6 : : *
7 : : * Permission is hereby granted, free of charge, to any person obtaining a
8 : : * copy of this software and associated documentation files (the "Software"),
9 : : * to deal in the Software without restriction, including without limitation
10 : : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
11 : : * and/or sell copies of the Software, and to permit persons to whom the
12 : : * Software is furnished to do so, subject to the following conditions:
13 : : *
14 : : * The above copyright notice and this permission notice (including the next
15 : : * paragraph) shall be included in all copies or substantial portions of the
16 : : * Software.
17 : : *
18 : : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
19 : : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
20 : : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
21 : : * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
22 : : * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
23 : : * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
24 : : * OTHER DEALINGS IN THE SOFTWARE.
25 : : */
26 : :
27 : : #include <linux/export.h>
28 : : #include <linux/moduleparam.h>
29 : :
30 : : #include <drm/drm_crtc.h>
31 : : #include <drm/drm_drv.h>
32 : : #include <drm/drm_framebuffer.h>
33 : : #include <drm/drm_print.h>
34 : : #include <drm/drm_vblank.h>
35 : :
36 : : #include "drm_internal.h"
37 : : #include "drm_trace.h"
38 : :
39 : : /**
40 : : * DOC: vblank handling
41 : : *
42 : : * Vertical blanking plays a major role in graphics rendering. To achieve
43 : : * tear-free display, users must synchronize page flips and/or rendering to
44 : : * vertical blanking. The DRM API offers ioctls to perform page flips
45 : : * synchronized to vertical blanking and wait for vertical blanking.
46 : : *
47 : : * The DRM core handles most of the vertical blanking management logic, which
48 : : * involves filtering out spurious interrupts, keeping race-free blanking
49 : : * counters, coping with counter wrap-around and resets and keeping use counts.
50 : : * It relies on the driver to generate vertical blanking interrupts and
51 : : * optionally provide a hardware vertical blanking counter.
52 : : *
53 : : * Drivers must initialize the vertical blanking handling core with a call to
54 : : * drm_vblank_init(). Minimally, a driver needs to implement
55 : : * &drm_crtc_funcs.enable_vblank and &drm_crtc_funcs.disable_vblank plus call
56 : : * drm_crtc_handle_vblank() in its vblank interrupt handler for working vblank
57 : : * support.
58 : : *
59 : : * Vertical blanking interrupts can be enabled by the DRM core or by drivers
60 : : * themselves (for instance to handle page flipping operations). The DRM core
61 : : * maintains a vertical blanking use count to ensure that the interrupts are not
62 : : * disabled while a user still needs them. To increment the use count, drivers
63 : : * call drm_crtc_vblank_get() and release the vblank reference again with
64 : : * drm_crtc_vblank_put(). In between these two calls vblank interrupts are
65 : : * guaranteed to be enabled.
66 : : *
67 : : * On many hardware disabling the vblank interrupt cannot be done in a race-free
68 : : * manner, see &drm_driver.vblank_disable_immediate and
69 : : * &drm_driver.max_vblank_count. In that case the vblank core only disables the
70 : : * vblanks after a timer has expired, which can be configured through the
71 : : * ``vblankoffdelay`` module parameter.
72 : : */
73 : :
74 : : /* Retry timestamp calculation up to 3 times to satisfy
75 : : * drm_timestamp_precision before giving up.
76 : : */
77 : : #define DRM_TIMESTAMP_MAXRETRIES 3
78 : :
79 : : /* Threshold in nanoseconds for detection of redundant
80 : : * vblank irq in drm_handle_vblank(). 1 msec should be ok.
81 : : */
82 : : #define DRM_REDUNDANT_VBLIRQ_THRESH_NS 1000000
83 : :
84 : : static bool
85 : : drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
86 : : ktime_t *tvblank, bool in_vblank_irq);
87 : :
88 : : static unsigned int drm_timestamp_precision = 20; /* Default to 20 usecs. */
89 : :
90 : : static int drm_vblank_offdelay = 5000; /* Default to 5000 msecs. */
91 : :
92 : : module_param_named(vblankoffdelay, drm_vblank_offdelay, int, 0600);
93 : : module_param_named(timestamp_precision_usec, drm_timestamp_precision, int, 0600);
94 : : MODULE_PARM_DESC(vblankoffdelay, "Delay until vblank irq auto-disable [msecs] (0: never disable, <0: disable immediately)");
95 : : MODULE_PARM_DESC(timestamp_precision_usec, "Max. error on timestamps [usecs]");
96 : :
97 : 0 : static void store_vblank(struct drm_device *dev, unsigned int pipe,
98 : : u32 vblank_count_inc,
99 : : ktime_t t_vblank, u32 last)
100 : : {
101 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
102 : :
103 [ # # ]: 0 : assert_spin_locked(&dev->vblank_time_lock);
104 : :
105 : 0 : vblank->last = last;
106 : :
107 : 0 : write_seqlock(&vblank->seqlock);
108 : 0 : vblank->time = t_vblank;
109 : 0 : atomic64_add(vblank_count_inc, &vblank->count);
110 : 0 : write_sequnlock(&vblank->seqlock);
111 : 0 : }
112 : :
113 : 0 : static u32 drm_max_vblank_count(struct drm_device *dev, unsigned int pipe)
114 : : {
115 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
116 : :
117 : 0 : return vblank->max_vblank_count ?: dev->max_vblank_count;
118 : : }
119 : :
120 : : /*
121 : : * "No hw counter" fallback implementation of .get_vblank_counter() hook,
122 : : * if there is no useable hardware frame counter available.
123 : : */
124 : 0 : static u32 drm_vblank_no_hw_counter(struct drm_device *dev, unsigned int pipe)
125 : : {
126 [ # # ]: 0 : WARN_ON_ONCE(drm_max_vblank_count(dev, pipe) != 0);
127 : : return 0;
128 : : }
129 : :
130 : 0 : static u32 __get_vblank_counter(struct drm_device *dev, unsigned int pipe)
131 : : {
132 [ # # ]: 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
133 : 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
134 : :
135 [ # # # # ]: 0 : if (WARN_ON(!crtc))
136 : : return 0;
137 : :
138 [ # # ]: 0 : if (crtc->funcs->get_vblank_counter)
139 : 0 : return crtc->funcs->get_vblank_counter(crtc);
140 : : }
141 : :
142 [ # # ]: 0 : if (dev->driver->get_vblank_counter)
143 : 0 : return dev->driver->get_vblank_counter(dev, pipe);
144 : :
145 [ # # ]: 0 : return drm_vblank_no_hw_counter(dev, pipe);
146 : : }
147 : :
148 : : /*
149 : : * Reset the stored timestamp for the current vblank count to correspond
150 : : * to the last vblank occurred.
151 : : *
152 : : * Only to be called from drm_crtc_vblank_on().
153 : : *
154 : : * Note: caller must hold &drm_device.vbl_lock since this reads & writes
155 : : * device vblank fields.
156 : : */
157 : 0 : static void drm_reset_vblank_timestamp(struct drm_device *dev, unsigned int pipe)
158 : : {
159 : 0 : u32 cur_vblank;
160 : 0 : bool rc;
161 : 0 : ktime_t t_vblank;
162 : 0 : int count = DRM_TIMESTAMP_MAXRETRIES;
163 : :
164 : 0 : spin_lock(&dev->vblank_time_lock);
165 : :
166 : : /*
167 : : * sample the current counter to avoid random jumps
168 : : * when drm_vblank_enable() applies the diff
169 : : */
170 : 0 : do {
171 : 0 : cur_vblank = __get_vblank_counter(dev, pipe);
172 : 0 : rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
173 [ # # # # ]: 0 : } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
174 : :
175 : : /*
176 : : * Only reinitialize corresponding vblank timestamp if high-precision query
177 : : * available and didn't fail. Otherwise reinitialize delayed at next vblank
178 : : * interrupt and assign 0 for now, to mark the vblanktimestamp as invalid.
179 : : */
180 [ # # ]: 0 : if (!rc)
181 : 0 : t_vblank = 0;
182 : :
183 : : /*
184 : : * +1 to make sure user will never see the same
185 : : * vblank counter value before and after a modeset
186 : : */
187 : 0 : store_vblank(dev, pipe, 1, t_vblank, cur_vblank);
188 : :
189 : 0 : spin_unlock(&dev->vblank_time_lock);
190 : 0 : }
191 : :
192 : : /*
193 : : * Call back into the driver to update the appropriate vblank counter
194 : : * (specified by @pipe). Deal with wraparound, if it occurred, and
195 : : * update the last read value so we can deal with wraparound on the next
196 : : * call if necessary.
197 : : *
198 : : * Only necessary when going from off->on, to account for frames we
199 : : * didn't get an interrupt for.
200 : : *
201 : : * Note: caller must hold &drm_device.vbl_lock since this reads & writes
202 : : * device vblank fields.
203 : : */
204 : 0 : static void drm_update_vblank_count(struct drm_device *dev, unsigned int pipe,
205 : : bool in_vblank_irq)
206 : : {
207 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
208 : 0 : u32 cur_vblank, diff;
209 : 0 : bool rc;
210 : 0 : ktime_t t_vblank;
211 : 0 : int count = DRM_TIMESTAMP_MAXRETRIES;
212 : 0 : int framedur_ns = vblank->framedur_ns;
213 [ # # ]: 0 : u32 max_vblank_count = drm_max_vblank_count(dev, pipe);
214 : :
215 : : /*
216 : : * Interrupts were disabled prior to this call, so deal with counter
217 : : * wrap if needed.
218 : : * NOTE! It's possible we lost a full dev->max_vblank_count + 1 events
219 : : * here if the register is small or we had vblank interrupts off for
220 : : * a long time.
221 : : *
222 : : * We repeat the hardware vblank counter & timestamp query until
223 : : * we get consistent results. This to prevent races between gpu
224 : : * updating its hardware counter while we are retrieving the
225 : : * corresponding vblank timestamp.
226 : : */
227 : 0 : do {
228 : 0 : cur_vblank = __get_vblank_counter(dev, pipe);
229 : 0 : rc = drm_get_last_vbltimestamp(dev, pipe, &t_vblank, in_vblank_irq);
230 [ # # # # ]: 0 : } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
231 : :
232 [ # # ]: 0 : if (max_vblank_count) {
233 : : /* trust the hw counter when it's around */
234 : 0 : diff = (cur_vblank - vblank->last) & max_vblank_count;
235 [ # # ]: 0 : } else if (rc && framedur_ns) {
236 : 0 : u64 diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
237 : :
238 : : /*
239 : : * Figure out how many vblanks we've missed based
240 : : * on the difference in the timestamps and the
241 : : * frame/field duration.
242 : : */
243 : :
244 : 0 : DRM_DEBUG_VBL("crtc %u: Calculating number of vblanks."
245 : : " diff_ns = %lld, framedur_ns = %d)\n",
246 : : pipe, (long long) diff_ns, framedur_ns);
247 : :
248 : 0 : diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
249 : :
250 [ # # ]: 0 : if (diff == 0 && in_vblank_irq)
251 : 0 : DRM_DEBUG_VBL("crtc %u: Redundant vblirq ignored\n",
252 : : pipe);
253 : : } else {
254 : : /* some kind of default for drivers w/o accurate vbl timestamping */
255 : 0 : diff = in_vblank_irq ? 1 : 0;
256 : : }
257 : :
258 : : /*
259 : : * Within a drm_vblank_pre_modeset - drm_vblank_post_modeset
260 : : * interval? If so then vblank irqs keep running and it will likely
261 : : * happen that the hardware vblank counter is not trustworthy as it
262 : : * might reset at some point in that interval and vblank timestamps
263 : : * are not trustworthy either in that interval. Iow. this can result
264 : : * in a bogus diff >> 1 which must be avoided as it would cause
265 : : * random large forward jumps of the software vblank counter.
266 : : */
267 [ # # # # ]: 0 : if (diff > 1 && (vblank->inmodeset & 0x2)) {
268 : 0 : DRM_DEBUG_VBL("clamping vblank bump to 1 on crtc %u: diffr=%u"
269 : : " due to pre-modeset.\n", pipe, diff);
270 : 0 : diff = 1;
271 : : }
272 : :
273 : 0 : DRM_DEBUG_VBL("updating vblank count on crtc %u:"
274 : : " current=%llu, diff=%u, hw=%u hw_last=%u\n",
275 : : pipe, atomic64_read(&vblank->count), diff,
276 : : cur_vblank, vblank->last);
277 : :
278 [ # # ]: 0 : if (diff == 0) {
279 [ # # ]: 0 : WARN_ON_ONCE(cur_vblank != vblank->last);
280 : 0 : return;
281 : : }
282 : :
283 : : /*
284 : : * Only reinitialize corresponding vblank timestamp if high-precision query
285 : : * available and didn't fail, or we were called from the vblank interrupt.
286 : : * Otherwise reinitialize delayed at next vblank interrupt and assign 0
287 : : * for now, to mark the vblanktimestamp as invalid.
288 : : */
289 [ # # ]: 0 : if (!rc && !in_vblank_irq)
290 : 0 : t_vblank = 0;
291 : :
292 : 0 : store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
293 : : }
294 : :
295 : : static u64 drm_vblank_count(struct drm_device *dev, unsigned int pipe)
296 : : {
297 : : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
298 : : u64 count;
299 : :
300 : : if (WARN_ON(pipe >= dev->num_crtcs))
301 : : return 0;
302 : :
303 : : count = atomic64_read(&vblank->count);
304 : :
305 : : /*
306 : : * This read barrier corresponds to the implicit write barrier of the
307 : : * write seqlock in store_vblank(). Note that this is the only place
308 : : * where we need an explicit barrier, since all other access goes
309 : : * through drm_vblank_count_and_time(), which already has the required
310 : : * read barrier curtesy of the read seqlock.
311 : : */
312 : : smp_rmb();
313 : :
314 : : return count;
315 : : }
316 : :
317 : : /**
318 : : * drm_crtc_accurate_vblank_count - retrieve the master vblank counter
319 : : * @crtc: which counter to retrieve
320 : : *
321 : : * This function is similar to drm_crtc_vblank_count() but this function
322 : : * interpolates to handle a race with vblank interrupts using the high precision
323 : : * timestamping support.
324 : : *
325 : : * This is mostly useful for hardware that can obtain the scanout position, but
326 : : * doesn't have a hardware frame counter.
327 : : */
328 : 0 : u64 drm_crtc_accurate_vblank_count(struct drm_crtc *crtc)
329 : : {
330 : 0 : struct drm_device *dev = crtc->dev;
331 [ # # ]: 0 : unsigned int pipe = drm_crtc_index(crtc);
332 : 0 : u64 vblank;
333 : 0 : unsigned long flags;
334 : :
335 [ # # # # : 0 : WARN_ONCE(drm_debug_enabled(DRM_UT_VBL) && !dev->driver->get_vblank_timestamp,
# # # # ]
336 : : "This function requires support for accurate vblank timestamps.");
337 : :
338 : 0 : spin_lock_irqsave(&dev->vblank_time_lock, flags);
339 : :
340 : 0 : drm_update_vblank_count(dev, pipe, false);
341 : 0 : vblank = drm_vblank_count(dev, pipe);
342 : :
343 : 0 : spin_unlock_irqrestore(&dev->vblank_time_lock, flags);
344 : :
345 : 0 : return vblank;
346 : : }
347 : : EXPORT_SYMBOL(drm_crtc_accurate_vblank_count);
348 : :
349 : 0 : static void __disable_vblank(struct drm_device *dev, unsigned int pipe)
350 : : {
351 [ # # ]: 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
352 : 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
353 : :
354 [ # # # # ]: 0 : if (WARN_ON(!crtc))
355 : : return;
356 : :
357 [ # # ]: 0 : if (crtc->funcs->disable_vblank) {
358 : 0 : crtc->funcs->disable_vblank(crtc);
359 : 0 : return;
360 : : }
361 : : }
362 : :
363 : 0 : dev->driver->disable_vblank(dev, pipe);
364 : : }
365 : :
366 : : /*
367 : : * Disable vblank irq's on crtc, make sure that last vblank count
368 : : * of hardware and corresponding consistent software vblank counter
369 : : * are preserved, even if there are any spurious vblank irq's after
370 : : * disable.
371 : : */
372 : 0 : void drm_vblank_disable_and_save(struct drm_device *dev, unsigned int pipe)
373 : : {
374 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
375 : 0 : unsigned long irqflags;
376 : :
377 [ # # ]: 0 : assert_spin_locked(&dev->vbl_lock);
378 : :
379 : : /* Prevent vblank irq processing while disabling vblank irqs,
380 : : * so no updates of timestamps or count can happen after we've
381 : : * disabled. Needed to prevent races in case of delayed irq's.
382 : : */
383 : 0 : spin_lock_irqsave(&dev->vblank_time_lock, irqflags);
384 : :
385 : : /*
386 : : * Update vblank count and disable vblank interrupts only if the
387 : : * interrupts were enabled. This avoids calling the ->disable_vblank()
388 : : * operation in atomic context with the hardware potentially runtime
389 : : * suspended.
390 : : */
391 [ # # ]: 0 : if (!vblank->enabled)
392 : 0 : goto out;
393 : :
394 : : /*
395 : : * Update the count and timestamp to maintain the
396 : : * appearance that the counter has been ticking all along until
397 : : * this time. This makes the count account for the entire time
398 : : * between drm_crtc_vblank_on() and drm_crtc_vblank_off().
399 : : */
400 : 0 : drm_update_vblank_count(dev, pipe, false);
401 : 0 : __disable_vblank(dev, pipe);
402 : 0 : vblank->enabled = false;
403 : :
404 : 0 : out:
405 : 0 : spin_unlock_irqrestore(&dev->vblank_time_lock, irqflags);
406 : 0 : }
407 : :
408 : 0 : static void vblank_disable_fn(struct timer_list *t)
409 : : {
410 : 0 : struct drm_vblank_crtc *vblank = from_timer(vblank, t, disable_timer);
411 : 0 : struct drm_device *dev = vblank->dev;
412 : 0 : unsigned int pipe = vblank->pipe;
413 : 0 : unsigned long irqflags;
414 : :
415 : 0 : spin_lock_irqsave(&dev->vbl_lock, irqflags);
416 [ # # # # ]: 0 : if (atomic_read(&vblank->refcount) == 0 && vblank->enabled) {
417 : 0 : DRM_DEBUG("disabling vblank on crtc %u\n", pipe);
418 : 0 : drm_vblank_disable_and_save(dev, pipe);
419 : : }
420 : 0 : spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
421 : 0 : }
422 : :
423 : 0 : void drm_vblank_cleanup(struct drm_device *dev)
424 : : {
425 : 0 : unsigned int pipe;
426 : :
427 : : /* Bail if the driver didn't call drm_vblank_init() */
428 [ # # ]: 0 : if (dev->num_crtcs == 0)
429 : : return;
430 : :
431 [ # # ]: 0 : for (pipe = 0; pipe < dev->num_crtcs; pipe++) {
432 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
433 : :
434 [ # # # # : 0 : WARN_ON(READ_ONCE(vblank->enabled) &&
# # ]
435 : : drm_core_check_feature(dev, DRIVER_MODESET));
436 : :
437 : 0 : del_timer_sync(&vblank->disable_timer);
438 : : }
439 : :
440 : 0 : kfree(dev->vblank);
441 : :
442 : 0 : dev->num_crtcs = 0;
443 : : }
444 : :
445 : : /**
446 : : * drm_vblank_init - initialize vblank support
447 : : * @dev: DRM device
448 : : * @num_crtcs: number of CRTCs supported by @dev
449 : : *
450 : : * This function initializes vblank support for @num_crtcs display pipelines.
451 : : * Cleanup is handled by the DRM core, or through calling drm_dev_fini() for
452 : : * drivers with a &drm_driver.release callback.
453 : : *
454 : : * Returns:
455 : : * Zero on success or a negative error code on failure.
456 : : */
457 : 0 : int drm_vblank_init(struct drm_device *dev, unsigned int num_crtcs)
458 : : {
459 : 0 : int ret = -ENOMEM;
460 : 0 : unsigned int i;
461 : :
462 : 0 : spin_lock_init(&dev->vbl_lock);
463 : 0 : spin_lock_init(&dev->vblank_time_lock);
464 : :
465 : 0 : dev->num_crtcs = num_crtcs;
466 : :
467 : 0 : dev->vblank = kcalloc(num_crtcs, sizeof(*dev->vblank), GFP_KERNEL);
468 [ # # ]: 0 : if (!dev->vblank)
469 : 0 : goto err;
470 : :
471 [ # # ]: 0 : for (i = 0; i < num_crtcs; i++) {
472 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[i];
473 : :
474 : 0 : vblank->dev = dev;
475 : 0 : vblank->pipe = i;
476 : 0 : init_waitqueue_head(&vblank->queue);
477 : 0 : timer_setup(&vblank->disable_timer, vblank_disable_fn, 0);
478 : 0 : seqlock_init(&vblank->seqlock);
479 : : }
480 : :
481 : 0 : DRM_INFO("Supports vblank timestamp caching Rev 2 (21.10.2013).\n");
482 : :
483 : : /* Driver specific high-precision vblank timestamping supported? */
484 [ # # ]: 0 : if (dev->driver->get_vblank_timestamp)
485 : 0 : DRM_INFO("Driver supports precise vblank timestamp query.\n");
486 : : else
487 : 0 : DRM_INFO("No driver support for vblank timestamp query.\n");
488 : :
489 : : /* Must have precise timestamping for reliable vblank instant disable */
490 [ # # # # ]: 0 : if (dev->vblank_disable_immediate && !dev->driver->get_vblank_timestamp) {
491 : 0 : dev->vblank_disable_immediate = false;
492 : 0 : DRM_INFO("Setting vblank_disable_immediate to false because "
493 : : "get_vblank_timestamp == NULL\n");
494 : : }
495 : :
496 : : return 0;
497 : :
498 : : err:
499 : 0 : dev->num_crtcs = 0;
500 : 0 : return ret;
501 : : }
502 : : EXPORT_SYMBOL(drm_vblank_init);
503 : :
504 : : /**
505 : : * drm_crtc_vblank_waitqueue - get vblank waitqueue for the CRTC
506 : : * @crtc: which CRTC's vblank waitqueue to retrieve
507 : : *
508 : : * This function returns a pointer to the vblank waitqueue for the CRTC.
509 : : * Drivers can use this to implement vblank waits using wait_event() and related
510 : : * functions.
511 : : */
512 : 0 : wait_queue_head_t *drm_crtc_vblank_waitqueue(struct drm_crtc *crtc)
513 : : {
514 : 0 : return &crtc->dev->vblank[drm_crtc_index(crtc)].queue;
515 : : }
516 : : EXPORT_SYMBOL(drm_crtc_vblank_waitqueue);
517 : :
518 : :
519 : : /**
520 : : * drm_calc_timestamping_constants - calculate vblank timestamp constants
521 : : * @crtc: drm_crtc whose timestamp constants should be updated.
522 : : * @mode: display mode containing the scanout timings
523 : : *
524 : : * Calculate and store various constants which are later needed by vblank and
525 : : * swap-completion timestamping, e.g, by
526 : : * drm_calc_vbltimestamp_from_scanoutpos(). They are derived from CRTC's true
527 : : * scanout timing, so they take things like panel scaling or other adjustments
528 : : * into account.
529 : : */
530 : 0 : void drm_calc_timestamping_constants(struct drm_crtc *crtc,
531 : : const struct drm_display_mode *mode)
532 : : {
533 : 0 : struct drm_device *dev = crtc->dev;
534 [ # # ]: 0 : unsigned int pipe = drm_crtc_index(crtc);
535 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
536 : 0 : int linedur_ns = 0, framedur_ns = 0;
537 : 0 : int dotclock = mode->crtc_clock;
538 : :
539 [ # # ]: 0 : if (!dev->num_crtcs)
540 : : return;
541 : :
542 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
543 : : return;
544 : :
545 : : /* Valid dotclock? */
546 [ # # ]: 0 : if (dotclock > 0) {
547 : 0 : int frame_size = mode->crtc_htotal * mode->crtc_vtotal;
548 : :
549 : : /*
550 : : * Convert scanline length in pixels and video
551 : : * dot clock to line duration and frame duration
552 : : * in nanoseconds:
553 : : */
554 [ # # ]: 0 : linedur_ns = div_u64((u64) mode->crtc_htotal * 1000000, dotclock);
555 : 0 : framedur_ns = div_u64((u64) frame_size * 1000000, dotclock);
556 : :
557 : : /*
558 : : * Fields of interlaced scanout modes are only half a frame duration.
559 : : */
560 [ # # ]: 0 : if (mode->flags & DRM_MODE_FLAG_INTERLACE)
561 : 0 : framedur_ns /= 2;
562 : : } else
563 : 0 : DRM_ERROR("crtc %u: Can't calculate constants, dotclock = 0!\n",
564 : : crtc->base.id);
565 : :
566 : 0 : vblank->linedur_ns = linedur_ns;
567 : 0 : vblank->framedur_ns = framedur_ns;
568 : 0 : vblank->hwmode = *mode;
569 : :
570 : 0 : DRM_DEBUG("crtc %u: hwmode: htotal %d, vtotal %d, vdisplay %d\n",
571 : : crtc->base.id, mode->crtc_htotal,
572 : : mode->crtc_vtotal, mode->crtc_vdisplay);
573 : 0 : DRM_DEBUG("crtc %u: clock %d kHz framedur %d linedur %d\n",
574 : : crtc->base.id, dotclock, framedur_ns, linedur_ns);
575 : : }
576 : : EXPORT_SYMBOL(drm_calc_timestamping_constants);
577 : :
578 : : /**
579 : : * drm_calc_vbltimestamp_from_scanoutpos - precise vblank timestamp helper
580 : : * @dev: DRM device
581 : : * @pipe: index of CRTC whose vblank timestamp to retrieve
582 : : * @max_error: Desired maximum allowable error in timestamps (nanosecs)
583 : : * On return contains true maximum error of timestamp
584 : : * @vblank_time: Pointer to time which should receive the timestamp
585 : : * @in_vblank_irq:
586 : : * True when called from drm_crtc_handle_vblank(). Some drivers
587 : : * need to apply some workarounds for gpu-specific vblank irq quirks
588 : : * if flag is set.
589 : : *
590 : : * Implements calculation of exact vblank timestamps from given drm_display_mode
591 : : * timings and current video scanout position of a CRTC. This can be directly
592 : : * used as the &drm_driver.get_vblank_timestamp implementation of a kms driver
593 : : * if &drm_driver.get_scanout_position is implemented.
594 : : *
595 : : * The current implementation only handles standard video modes. For double scan
596 : : * and interlaced modes the driver is supposed to adjust the hardware mode
597 : : * (taken from &drm_crtc_state.adjusted mode for atomic modeset drivers) to
598 : : * match the scanout position reported.
599 : : *
600 : : * Note that atomic drivers must call drm_calc_timestamping_constants() before
601 : : * enabling a CRTC. The atomic helpers already take care of that in
602 : : * drm_atomic_helper_update_legacy_modeset_state().
603 : : *
604 : : * Returns:
605 : : *
606 : : * Returns true on success, and false on failure, i.e. when no accurate
607 : : * timestamp could be acquired.
608 : : */
609 : 0 : bool drm_calc_vbltimestamp_from_scanoutpos(struct drm_device *dev,
610 : : unsigned int pipe,
611 : : int *max_error,
612 : : ktime_t *vblank_time,
613 : : bool in_vblank_irq)
614 : : {
615 : 0 : struct timespec64 ts_etime, ts_vblank_time;
616 : 0 : ktime_t stime, etime;
617 : 0 : bool vbl_status;
618 : 0 : struct drm_crtc *crtc;
619 : 0 : const struct drm_display_mode *mode;
620 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
621 : 0 : int vpos, hpos, i;
622 : 0 : int delta_ns, duration_ns;
623 : :
624 [ # # ]: 0 : if (!drm_core_check_feature(dev, DRIVER_MODESET))
625 : : return false;
626 : :
627 : 0 : crtc = drm_crtc_from_index(dev, pipe);
628 : :
629 [ # # # # ]: 0 : if (pipe >= dev->num_crtcs || !crtc) {
630 : 0 : DRM_ERROR("Invalid crtc %u\n", pipe);
631 : 0 : return false;
632 : : }
633 : :
634 : : /* Scanout position query not supported? Should not happen. */
635 [ # # ]: 0 : if (!dev->driver->get_scanout_position) {
636 : 0 : DRM_ERROR("Called from driver w/o get_scanout_position()!?\n");
637 : 0 : return false;
638 : : }
639 : :
640 [ # # # # ]: 0 : if (drm_drv_uses_atomic_modeset(dev))
641 : 0 : mode = &vblank->hwmode;
642 : : else
643 : 0 : mode = &crtc->hwmode;
644 : :
645 : : /* If mode timing undefined, just return as no-op:
646 : : * Happens during initial modesetting of a crtc.
647 : : */
648 [ # # ]: 0 : if (mode->crtc_clock == 0) {
649 : 0 : DRM_DEBUG("crtc %u: Noop due to uninitialized mode.\n", pipe);
650 [ # # # # ]: 0 : WARN_ON_ONCE(drm_drv_uses_atomic_modeset(dev));
651 : :
652 : : return false;
653 : : }
654 : :
655 : : /* Get current scanout position with system timestamp.
656 : : * Repeat query up to DRM_TIMESTAMP_MAXRETRIES times
657 : : * if single query takes longer than max_error nanoseconds.
658 : : *
659 : : * This guarantees a tight bound on maximum error if
660 : : * code gets preempted or delayed for some reason.
661 : : */
662 [ # # ]: 0 : for (i = 0; i < DRM_TIMESTAMP_MAXRETRIES; i++) {
663 : : /*
664 : : * Get vertical and horizontal scanout position vpos, hpos,
665 : : * and bounding timestamps stime, etime, pre/post query.
666 : : */
667 : 0 : vbl_status = dev->driver->get_scanout_position(dev, pipe,
668 : : in_vblank_irq,
669 : : &vpos, &hpos,
670 : : &stime, &etime,
671 : : mode);
672 : :
673 : : /* Return as no-op if scanout query unsupported or failed. */
674 [ # # ]: 0 : if (!vbl_status) {
675 : 0 : DRM_DEBUG("crtc %u : scanoutpos query failed.\n",
676 : : pipe);
677 : 0 : return false;
678 : : }
679 : :
680 : : /* Compute uncertainty in timestamp of scanout position query. */
681 [ # # ]: 0 : duration_ns = ktime_to_ns(etime) - ktime_to_ns(stime);
682 : :
683 : : /* Accept result with < max_error nsecs timing uncertainty. */
684 [ # # ]: 0 : if (duration_ns <= *max_error)
685 : : break;
686 : : }
687 : :
688 : : /* Noisy system timing? */
689 [ # # ]: 0 : if (i == DRM_TIMESTAMP_MAXRETRIES) {
690 : 0 : DRM_DEBUG("crtc %u: Noisy timestamp %d us > %d us [%d reps].\n",
691 : : pipe, duration_ns/1000, *max_error/1000, i);
692 : : }
693 : :
694 : : /* Return upper bound of timestamp precision error. */
695 : 0 : *max_error = duration_ns;
696 : :
697 : : /* Convert scanout position into elapsed time at raw_time query
698 : : * since start of scanout at first display scanline. delta_ns
699 : : * can be negative if start of scanout hasn't happened yet.
700 : : */
701 [ # # ]: 0 : delta_ns = div_s64(1000000LL * (vpos * mode->crtc_htotal + hpos),
702 : : mode->crtc_clock);
703 : :
704 : : /* Subtract time delta from raw timestamp to get final
705 : : * vblank_time timestamp for end of vblank.
706 : : */
707 : 0 : *vblank_time = ktime_sub_ns(etime, delta_ns);
708 : :
709 [ # # ]: 0 : if (!drm_debug_enabled(DRM_UT_VBL))
710 : : return true;
711 : :
712 : 0 : ts_etime = ktime_to_timespec64(etime);
713 : 0 : ts_vblank_time = ktime_to_timespec64(*vblank_time);
714 : :
715 : 0 : DRM_DEBUG_VBL("crtc %u : v p(%d,%d)@ %lld.%06ld -> %lld.%06ld [e %d us, %d rep]\n",
716 : : pipe, hpos, vpos,
717 : : (u64)ts_etime.tv_sec, ts_etime.tv_nsec / 1000,
718 : : (u64)ts_vblank_time.tv_sec, ts_vblank_time.tv_nsec / 1000,
719 : : duration_ns / 1000, i);
720 : :
721 : 0 : return true;
722 : : }
723 : : EXPORT_SYMBOL(drm_calc_vbltimestamp_from_scanoutpos);
724 : :
725 : : /**
726 : : * drm_get_last_vbltimestamp - retrieve raw timestamp for the most recent
727 : : * vblank interval
728 : : * @dev: DRM device
729 : : * @pipe: index of CRTC whose vblank timestamp to retrieve
730 : : * @tvblank: Pointer to target time which should receive the timestamp
731 : : * @in_vblank_irq:
732 : : * True when called from drm_crtc_handle_vblank(). Some drivers
733 : : * need to apply some workarounds for gpu-specific vblank irq quirks
734 : : * if flag is set.
735 : : *
736 : : * Fetches the system timestamp corresponding to the time of the most recent
737 : : * vblank interval on specified CRTC. May call into kms-driver to
738 : : * compute the timestamp with a high-precision GPU specific method.
739 : : *
740 : : * Returns zero if timestamp originates from uncorrected do_gettimeofday()
741 : : * call, i.e., it isn't very precisely locked to the true vblank.
742 : : *
743 : : * Returns:
744 : : * True if timestamp is considered to be very precise, false otherwise.
745 : : */
746 : : static bool
747 : 0 : drm_get_last_vbltimestamp(struct drm_device *dev, unsigned int pipe,
748 : : ktime_t *tvblank, bool in_vblank_irq)
749 : : {
750 : 0 : bool ret = false;
751 : :
752 : : /* Define requested maximum error on timestamps (nanoseconds). */
753 : 0 : int max_error = (int) drm_timestamp_precision * 1000;
754 : :
755 : : /* Query driver if possible and precision timestamping enabled. */
756 [ # # # # ]: 0 : if (dev->driver->get_vblank_timestamp && (max_error > 0))
757 : 0 : ret = dev->driver->get_vblank_timestamp(dev, pipe, &max_error,
758 : : tvblank, in_vblank_irq);
759 : :
760 : : /* GPU high precision timestamp query unsupported or failed.
761 : : * Return current monotonic/gettimeofday timestamp as best estimate.
762 : : */
763 [ # # ]: 0 : if (!ret)
764 : 0 : *tvblank = ktime_get();
765 : :
766 : 0 : return ret;
767 : : }
768 : :
769 : : /**
770 : : * drm_crtc_vblank_count - retrieve "cooked" vblank counter value
771 : : * @crtc: which counter to retrieve
772 : : *
773 : : * Fetches the "cooked" vblank count value that represents the number of
774 : : * vblank events since the system was booted, including lost events due to
775 : : * modesetting activity. Note that this timer isn't correct against a racing
776 : : * vblank interrupt (since it only reports the software vblank counter), see
777 : : * drm_crtc_accurate_vblank_count() for such use-cases.
778 : : *
779 : : * Note that for a given vblank counter value drm_crtc_handle_vblank()
780 : : * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
781 : : * provide a barrier: Any writes done before calling
782 : : * drm_crtc_handle_vblank() will be visible to callers of the later
783 : : * functions, iff the vblank count is the same or a later one.
784 : : *
785 : : * See also &drm_vblank_crtc.count.
786 : : *
787 : : * Returns:
788 : : * The software vblank counter.
789 : : */
790 : 0 : u64 drm_crtc_vblank_count(struct drm_crtc *crtc)
791 : : {
792 : 0 : return drm_vblank_count(crtc->dev, drm_crtc_index(crtc));
793 : : }
794 : : EXPORT_SYMBOL(drm_crtc_vblank_count);
795 : :
796 : : /**
797 : : * drm_vblank_count_and_time - retrieve "cooked" vblank counter value and the
798 : : * system timestamp corresponding to that vblank counter value.
799 : : * @dev: DRM device
800 : : * @pipe: index of CRTC whose counter to retrieve
801 : : * @vblanktime: Pointer to ktime_t to receive the vblank timestamp.
802 : : *
803 : : * Fetches the "cooked" vblank count value that represents the number of
804 : : * vblank events since the system was booted, including lost events due to
805 : : * modesetting activity. Returns corresponding system timestamp of the time
806 : : * of the vblank interval that corresponds to the current vblank counter value.
807 : : *
808 : : * This is the legacy version of drm_crtc_vblank_count_and_time().
809 : : */
810 : : static u64 drm_vblank_count_and_time(struct drm_device *dev, unsigned int pipe,
811 : : ktime_t *vblanktime)
812 : : {
813 : : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
814 : : u64 vblank_count;
815 : : unsigned int seq;
816 : :
817 : : if (WARN_ON(pipe >= dev->num_crtcs)) {
818 : : *vblanktime = 0;
819 : : return 0;
820 : : }
821 : :
822 : : do {
823 : : seq = read_seqbegin(&vblank->seqlock);
824 : : vblank_count = atomic64_read(&vblank->count);
825 : : *vblanktime = vblank->time;
826 : : } while (read_seqretry(&vblank->seqlock, seq));
827 : :
828 : : return vblank_count;
829 : : }
830 : :
831 : : /**
832 : : * drm_crtc_vblank_count_and_time - retrieve "cooked" vblank counter value
833 : : * and the system timestamp corresponding to that vblank counter value
834 : : * @crtc: which counter to retrieve
835 : : * @vblanktime: Pointer to time to receive the vblank timestamp.
836 : : *
837 : : * Fetches the "cooked" vblank count value that represents the number of
838 : : * vblank events since the system was booted, including lost events due to
839 : : * modesetting activity. Returns corresponding system timestamp of the time
840 : : * of the vblank interval that corresponds to the current vblank counter value.
841 : : *
842 : : * Note that for a given vblank counter value drm_crtc_handle_vblank()
843 : : * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
844 : : * provide a barrier: Any writes done before calling
845 : : * drm_crtc_handle_vblank() will be visible to callers of the later
846 : : * functions, iff the vblank count is the same or a later one.
847 : : *
848 : : * See also &drm_vblank_crtc.count.
849 : : */
850 : 0 : u64 drm_crtc_vblank_count_and_time(struct drm_crtc *crtc,
851 : : ktime_t *vblanktime)
852 : : {
853 : 0 : return drm_vblank_count_and_time(crtc->dev, drm_crtc_index(crtc),
854 : : vblanktime);
855 : : }
856 : : EXPORT_SYMBOL(drm_crtc_vblank_count_and_time);
857 : :
858 : 0 : static void send_vblank_event(struct drm_device *dev,
859 : : struct drm_pending_vblank_event *e,
860 : : u64 seq, ktime_t now)
861 : : {
862 : 0 : struct timespec64 tv;
863 : :
864 [ # # # ]: 0 : switch (e->event.base.type) {
865 : 0 : case DRM_EVENT_VBLANK:
866 : : case DRM_EVENT_FLIP_COMPLETE:
867 : 0 : tv = ktime_to_timespec64(now);
868 : 0 : e->event.vbl.sequence = seq;
869 : : /*
870 : : * e->event is a user space structure, with hardcoded unsigned
871 : : * 32-bit seconds/microseconds. This is safe as we always use
872 : : * monotonic timestamps since linux-4.15
873 : : */
874 : 0 : e->event.vbl.tv_sec = tv.tv_sec;
875 : 0 : e->event.vbl.tv_usec = tv.tv_nsec / 1000;
876 : 0 : break;
877 : 0 : case DRM_EVENT_CRTC_SEQUENCE:
878 [ # # ]: 0 : if (seq)
879 : 0 : e->event.seq.sequence = seq;
880 : 0 : e->event.seq.time_ns = ktime_to_ns(now);
881 : 0 : break;
882 : : }
883 : 0 : trace_drm_vblank_event_delivered(e->base.file_priv, e->pipe, seq);
884 : 0 : drm_send_event_locked(dev, &e->base);
885 : 0 : }
886 : :
887 : : /**
888 : : * drm_crtc_arm_vblank_event - arm vblank event after pageflip
889 : : * @crtc: the source CRTC of the vblank event
890 : : * @e: the event to send
891 : : *
892 : : * A lot of drivers need to generate vblank events for the very next vblank
893 : : * interrupt. For example when the page flip interrupt happens when the page
894 : : * flip gets armed, but not when it actually executes within the next vblank
895 : : * period. This helper function implements exactly the required vblank arming
896 : : * behaviour.
897 : : *
898 : : * NOTE: Drivers using this to send out the &drm_crtc_state.event as part of an
899 : : * atomic commit must ensure that the next vblank happens at exactly the same
900 : : * time as the atomic commit is committed to the hardware. This function itself
901 : : * does **not** protect against the next vblank interrupt racing with either this
902 : : * function call or the atomic commit operation. A possible sequence could be:
903 : : *
904 : : * 1. Driver commits new hardware state into vblank-synchronized registers.
905 : : * 2. A vblank happens, committing the hardware state. Also the corresponding
906 : : * vblank interrupt is fired off and fully processed by the interrupt
907 : : * handler.
908 : : * 3. The atomic commit operation proceeds to call drm_crtc_arm_vblank_event().
909 : : * 4. The event is only send out for the next vblank, which is wrong.
910 : : *
911 : : * An equivalent race can happen when the driver calls
912 : : * drm_crtc_arm_vblank_event() before writing out the new hardware state.
913 : : *
914 : : * The only way to make this work safely is to prevent the vblank from firing
915 : : * (and the hardware from committing anything else) until the entire atomic
916 : : * commit sequence has run to completion. If the hardware does not have such a
917 : : * feature (e.g. using a "go" bit), then it is unsafe to use this functions.
918 : : * Instead drivers need to manually send out the event from their interrupt
919 : : * handler by calling drm_crtc_send_vblank_event() and make sure that there's no
920 : : * possible race with the hardware committing the atomic update.
921 : : *
922 : : * Caller must hold a vblank reference for the event @e acquired by a
923 : : * drm_crtc_vblank_get(), which will be dropped when the next vblank arrives.
924 : : */
925 : 0 : void drm_crtc_arm_vblank_event(struct drm_crtc *crtc,
926 : : struct drm_pending_vblank_event *e)
927 : : {
928 : 0 : struct drm_device *dev = crtc->dev;
929 : 0 : unsigned int pipe = drm_crtc_index(crtc);
930 : :
931 [ # # ]: 0 : assert_spin_locked(&dev->event_lock);
932 : :
933 : 0 : e->pipe = pipe;
934 : 0 : e->sequence = drm_crtc_accurate_vblank_count(crtc) + 1;
935 : 0 : list_add_tail(&e->base.link, &dev->vblank_event_list);
936 : 0 : }
937 : : EXPORT_SYMBOL(drm_crtc_arm_vblank_event);
938 : :
939 : : /**
940 : : * drm_crtc_send_vblank_event - helper to send vblank event after pageflip
941 : : * @crtc: the source CRTC of the vblank event
942 : : * @e: the event to send
943 : : *
944 : : * Updates sequence # and timestamp on event for the most recently processed
945 : : * vblank, and sends it to userspace. Caller must hold event lock.
946 : : *
947 : : * See drm_crtc_arm_vblank_event() for a helper which can be used in certain
948 : : * situation, especially to send out events for atomic commit operations.
949 : : */
950 : 0 : void drm_crtc_send_vblank_event(struct drm_crtc *crtc,
951 : : struct drm_pending_vblank_event *e)
952 : : {
953 : 0 : struct drm_device *dev = crtc->dev;
954 : 0 : u64 seq;
955 [ # # ]: 0 : unsigned int pipe = drm_crtc_index(crtc);
956 : 0 : ktime_t now;
957 : :
958 [ # # ]: 0 : if (dev->num_crtcs > 0) {
959 : 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
960 : : } else {
961 : 0 : seq = 0;
962 : :
963 : 0 : now = ktime_get();
964 : : }
965 : 0 : e->pipe = pipe;
966 : 0 : send_vblank_event(dev, e, seq, now);
967 : 0 : }
968 : : EXPORT_SYMBOL(drm_crtc_send_vblank_event);
969 : :
970 : 0 : static int __enable_vblank(struct drm_device *dev, unsigned int pipe)
971 : : {
972 [ # # ]: 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
973 : 0 : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
974 : :
975 [ # # # # ]: 0 : if (WARN_ON(!crtc))
976 : : return 0;
977 : :
978 [ # # ]: 0 : if (crtc->funcs->enable_vblank)
979 : 0 : return crtc->funcs->enable_vblank(crtc);
980 : : }
981 : :
982 : 0 : return dev->driver->enable_vblank(dev, pipe);
983 : : }
984 : :
985 : 0 : static int drm_vblank_enable(struct drm_device *dev, unsigned int pipe)
986 : : {
987 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
988 : 0 : int ret = 0;
989 : :
990 [ # # ]: 0 : assert_spin_locked(&dev->vbl_lock);
991 : :
992 : 0 : spin_lock(&dev->vblank_time_lock);
993 : :
994 [ # # ]: 0 : if (!vblank->enabled) {
995 : : /*
996 : : * Enable vblank irqs under vblank_time_lock protection.
997 : : * All vblank count & timestamp updates are held off
998 : : * until we are done reinitializing master counter and
999 : : * timestamps. Filtercode in drm_handle_vblank() will
1000 : : * prevent double-accounting of same vblank interval.
1001 : : */
1002 : 0 : ret = __enable_vblank(dev, pipe);
1003 : 0 : DRM_DEBUG("enabling vblank on crtc %u, ret: %d\n", pipe, ret);
1004 [ # # ]: 0 : if (ret) {
1005 : 0 : atomic_dec(&vblank->refcount);
1006 : : } else {
1007 : 0 : drm_update_vblank_count(dev, pipe, 0);
1008 : : /* drm_update_vblank_count() includes a wmb so we just
1009 : : * need to ensure that the compiler emits the write
1010 : : * to mark the vblank as enabled after the call
1011 : : * to drm_update_vblank_count().
1012 : : */
1013 : 0 : WRITE_ONCE(vblank->enabled, true);
1014 : : }
1015 : : }
1016 : :
1017 : 0 : spin_unlock(&dev->vblank_time_lock);
1018 : :
1019 : 0 : return ret;
1020 : : }
1021 : :
1022 : 0 : static int drm_vblank_get(struct drm_device *dev, unsigned int pipe)
1023 : : {
1024 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1025 : 0 : unsigned long irqflags;
1026 : 0 : int ret = 0;
1027 : :
1028 [ # # ]: 0 : if (!dev->num_crtcs)
1029 : : return -EINVAL;
1030 : :
1031 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1032 : : return -EINVAL;
1033 : :
1034 : 0 : spin_lock_irqsave(&dev->vbl_lock, irqflags);
1035 : : /* Going from 0->1 means we have to enable interrupts again */
1036 [ # # ]: 0 : if (atomic_add_return(1, &vblank->refcount) == 1) {
1037 : 0 : ret = drm_vblank_enable(dev, pipe);
1038 : : } else {
1039 [ # # ]: 0 : if (!vblank->enabled) {
1040 : 0 : atomic_dec(&vblank->refcount);
1041 : 0 : ret = -EINVAL;
1042 : : }
1043 : : }
1044 : 0 : spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1045 : :
1046 : 0 : return ret;
1047 : : }
1048 : :
1049 : : /**
1050 : : * drm_crtc_vblank_get - get a reference count on vblank events
1051 : : * @crtc: which CRTC to own
1052 : : *
1053 : : * Acquire a reference count on vblank events to avoid having them disabled
1054 : : * while in use.
1055 : : *
1056 : : * Returns:
1057 : : * Zero on success or a negative error code on failure.
1058 : : */
1059 : 0 : int drm_crtc_vblank_get(struct drm_crtc *crtc)
1060 : : {
1061 : 0 : return drm_vblank_get(crtc->dev, drm_crtc_index(crtc));
1062 : : }
1063 : : EXPORT_SYMBOL(drm_crtc_vblank_get);
1064 : :
1065 : 0 : static void drm_vblank_put(struct drm_device *dev, unsigned int pipe)
1066 : : {
1067 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1068 : :
1069 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1070 : : return;
1071 : :
1072 [ # # # # ]: 0 : if (WARN_ON(atomic_read(&vblank->refcount) == 0))
1073 : : return;
1074 : :
1075 : : /* Last user schedules interrupt disable */
1076 [ # # ]: 0 : if (atomic_dec_and_test(&vblank->refcount)) {
1077 [ # # ]: 0 : if (drm_vblank_offdelay == 0)
1078 : : return;
1079 [ # # ]: 0 : else if (drm_vblank_offdelay < 0)
1080 : 0 : vblank_disable_fn(&vblank->disable_timer);
1081 [ # # ]: 0 : else if (!dev->vblank_disable_immediate)
1082 : 0 : mod_timer(&vblank->disable_timer,
1083 : 0 : jiffies + ((drm_vblank_offdelay * HZ)/1000));
1084 : : }
1085 : : }
1086 : :
1087 : : /**
1088 : : * drm_crtc_vblank_put - give up ownership of vblank events
1089 : : * @crtc: which counter to give up
1090 : : *
1091 : : * Release ownership of a given vblank counter, turning off interrupts
1092 : : * if possible. Disable interrupts after drm_vblank_offdelay milliseconds.
1093 : : */
1094 : 0 : void drm_crtc_vblank_put(struct drm_crtc *crtc)
1095 : : {
1096 : 0 : drm_vblank_put(crtc->dev, drm_crtc_index(crtc));
1097 : 0 : }
1098 : : EXPORT_SYMBOL(drm_crtc_vblank_put);
1099 : :
1100 : : /**
1101 : : * drm_wait_one_vblank - wait for one vblank
1102 : : * @dev: DRM device
1103 : : * @pipe: CRTC index
1104 : : *
1105 : : * This waits for one vblank to pass on @pipe, using the irq driver interfaces.
1106 : : * It is a failure to call this when the vblank irq for @pipe is disabled, e.g.
1107 : : * due to lack of driver support or because the crtc is off.
1108 : : *
1109 : : * This is the legacy version of drm_crtc_wait_one_vblank().
1110 : : */
1111 : 0 : void drm_wait_one_vblank(struct drm_device *dev, unsigned int pipe)
1112 : : {
1113 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1114 : 0 : int ret;
1115 : 0 : u64 last;
1116 : :
1117 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1118 : : return;
1119 : :
1120 : 0 : ret = drm_vblank_get(dev, pipe);
1121 [ # # # # ]: 0 : if (WARN(ret, "vblank not available on crtc %i, ret=%i\n", pipe, ret))
1122 : : return;
1123 : :
1124 : 0 : last = drm_vblank_count(dev, pipe);
1125 : :
1126 [ # # # # : 0 : ret = wait_event_timeout(vblank->queue,
# # ]
1127 : : last != drm_vblank_count(dev, pipe),
1128 : : msecs_to_jiffies(100));
1129 : :
1130 [ # # ]: 0 : WARN(ret == 0, "vblank wait timed out on crtc %i\n", pipe);
1131 : :
1132 : 0 : drm_vblank_put(dev, pipe);
1133 : : }
1134 : : EXPORT_SYMBOL(drm_wait_one_vblank);
1135 : :
1136 : : /**
1137 : : * drm_crtc_wait_one_vblank - wait for one vblank
1138 : : * @crtc: DRM crtc
1139 : : *
1140 : : * This waits for one vblank to pass on @crtc, using the irq driver interfaces.
1141 : : * It is a failure to call this when the vblank irq for @crtc is disabled, e.g.
1142 : : * due to lack of driver support or because the crtc is off.
1143 : : */
1144 : 0 : void drm_crtc_wait_one_vblank(struct drm_crtc *crtc)
1145 : : {
1146 : 0 : drm_wait_one_vblank(crtc->dev, drm_crtc_index(crtc));
1147 : 0 : }
1148 : : EXPORT_SYMBOL(drm_crtc_wait_one_vblank);
1149 : :
1150 : : /**
1151 : : * drm_crtc_vblank_off - disable vblank events on a CRTC
1152 : : * @crtc: CRTC in question
1153 : : *
1154 : : * Drivers can use this function to shut down the vblank interrupt handling when
1155 : : * disabling a crtc. This function ensures that the latest vblank frame count is
1156 : : * stored so that drm_vblank_on can restore it again.
1157 : : *
1158 : : * Drivers must use this function when the hardware vblank counter can get
1159 : : * reset, e.g. when suspending or disabling the @crtc in general.
1160 : : */
1161 : 0 : void drm_crtc_vblank_off(struct drm_crtc *crtc)
1162 : : {
1163 : 0 : struct drm_device *dev = crtc->dev;
1164 [ # # ]: 0 : unsigned int pipe = drm_crtc_index(crtc);
1165 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1166 : 0 : struct drm_pending_vblank_event *e, *t;
1167 : :
1168 : 0 : ktime_t now;
1169 : 0 : unsigned long irqflags;
1170 : 0 : u64 seq;
1171 : :
1172 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1173 : 0 : return;
1174 : :
1175 : 0 : spin_lock_irqsave(&dev->event_lock, irqflags);
1176 : :
1177 : 0 : spin_lock(&dev->vbl_lock);
1178 : 0 : DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1179 : : pipe, vblank->enabled, vblank->inmodeset);
1180 : :
1181 : : /* Avoid redundant vblank disables without previous
1182 : : * drm_crtc_vblank_on(). */
1183 [ # # # # ]: 0 : if (drm_core_check_feature(dev, DRIVER_ATOMIC) || !vblank->inmodeset)
1184 : 0 : drm_vblank_disable_and_save(dev, pipe);
1185 : :
1186 : 0 : wake_up(&vblank->queue);
1187 : :
1188 : : /*
1189 : : * Prevent subsequent drm_vblank_get() from re-enabling
1190 : : * the vblank interrupt by bumping the refcount.
1191 : : */
1192 [ # # ]: 0 : if (!vblank->inmodeset) {
1193 : 0 : atomic_inc(&vblank->refcount);
1194 : 0 : vblank->inmodeset = 1;
1195 : : }
1196 : 0 : spin_unlock(&dev->vbl_lock);
1197 : :
1198 : : /* Send any queued vblank events, lest the natives grow disquiet */
1199 : 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
1200 : :
1201 [ # # ]: 0 : list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1202 [ # # ]: 0 : if (e->pipe != pipe)
1203 : 0 : continue;
1204 : 0 : DRM_DEBUG("Sending premature vblank event on disable: "
1205 : : "wanted %llu, current %llu\n",
1206 : : e->sequence, seq);
1207 : 0 : list_del(&e->base.link);
1208 : 0 : drm_vblank_put(dev, pipe);
1209 : 0 : send_vblank_event(dev, e, seq, now);
1210 : : }
1211 : 0 : spin_unlock_irqrestore(&dev->event_lock, irqflags);
1212 : :
1213 : : /* Will be reset by the modeset helpers when re-enabling the crtc by
1214 : : * calling drm_calc_timestamping_constants(). */
1215 : 0 : vblank->hwmode.crtc_clock = 0;
1216 : : }
1217 : : EXPORT_SYMBOL(drm_crtc_vblank_off);
1218 : :
1219 : : /**
1220 : : * drm_crtc_vblank_reset - reset vblank state to off on a CRTC
1221 : : * @crtc: CRTC in question
1222 : : *
1223 : : * Drivers can use this function to reset the vblank state to off at load time.
1224 : : * Drivers should use this together with the drm_crtc_vblank_off() and
1225 : : * drm_crtc_vblank_on() functions. The difference compared to
1226 : : * drm_crtc_vblank_off() is that this function doesn't save the vblank counter
1227 : : * and hence doesn't need to call any driver hooks.
1228 : : *
1229 : : * This is useful for recovering driver state e.g. on driver load, or on resume.
1230 : : */
1231 : 0 : void drm_crtc_vblank_reset(struct drm_crtc *crtc)
1232 : : {
1233 : 0 : struct drm_device *dev = crtc->dev;
1234 : 0 : unsigned long irqflags;
1235 : 0 : unsigned int pipe = drm_crtc_index(crtc);
1236 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1237 : :
1238 : 0 : spin_lock_irqsave(&dev->vbl_lock, irqflags);
1239 : : /*
1240 : : * Prevent subsequent drm_vblank_get() from enabling the vblank
1241 : : * interrupt by bumping the refcount.
1242 : : */
1243 [ # # ]: 0 : if (!vblank->inmodeset) {
1244 : 0 : atomic_inc(&vblank->refcount);
1245 : 0 : vblank->inmodeset = 1;
1246 : : }
1247 : 0 : spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1248 : :
1249 [ # # ]: 0 : WARN_ON(!list_empty(&dev->vblank_event_list));
1250 : 0 : }
1251 : : EXPORT_SYMBOL(drm_crtc_vblank_reset);
1252 : :
1253 : : /**
1254 : : * drm_crtc_set_max_vblank_count - configure the hw max vblank counter value
1255 : : * @crtc: CRTC in question
1256 : : * @max_vblank_count: max hardware vblank counter value
1257 : : *
1258 : : * Update the maximum hardware vblank counter value for @crtc
1259 : : * at runtime. Useful for hardware where the operation of the
1260 : : * hardware vblank counter depends on the currently active
1261 : : * display configuration.
1262 : : *
1263 : : * For example, if the hardware vblank counter does not work
1264 : : * when a specific connector is active the maximum can be set
1265 : : * to zero. And when that specific connector isn't active the
1266 : : * maximum can again be set to the appropriate non-zero value.
1267 : : *
1268 : : * If used, must be called before drm_vblank_on().
1269 : : */
1270 : 0 : void drm_crtc_set_max_vblank_count(struct drm_crtc *crtc,
1271 : : u32 max_vblank_count)
1272 : : {
1273 : 0 : struct drm_device *dev = crtc->dev;
1274 [ # # ]: 0 : unsigned int pipe = drm_crtc_index(crtc);
1275 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1276 : :
1277 [ # # ]: 0 : WARN_ON(dev->max_vblank_count);
1278 [ # # ]: 0 : WARN_ON(!READ_ONCE(vblank->inmodeset));
1279 : :
1280 : 0 : vblank->max_vblank_count = max_vblank_count;
1281 : 0 : }
1282 : : EXPORT_SYMBOL(drm_crtc_set_max_vblank_count);
1283 : :
1284 : : /**
1285 : : * drm_crtc_vblank_on - enable vblank events on a CRTC
1286 : : * @crtc: CRTC in question
1287 : : *
1288 : : * This functions restores the vblank interrupt state captured with
1289 : : * drm_crtc_vblank_off() again and is generally called when enabling @crtc. Note
1290 : : * that calls to drm_crtc_vblank_on() and drm_crtc_vblank_off() can be
1291 : : * unbalanced and so can also be unconditionally called in driver load code to
1292 : : * reflect the current hardware state of the crtc.
1293 : : */
1294 : 0 : void drm_crtc_vblank_on(struct drm_crtc *crtc)
1295 : : {
1296 : 0 : struct drm_device *dev = crtc->dev;
1297 [ # # ]: 0 : unsigned int pipe = drm_crtc_index(crtc);
1298 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1299 : 0 : unsigned long irqflags;
1300 : :
1301 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1302 : : return;
1303 : :
1304 : 0 : spin_lock_irqsave(&dev->vbl_lock, irqflags);
1305 : 0 : DRM_DEBUG_VBL("crtc %d, vblank enabled %d, inmodeset %d\n",
1306 : : pipe, vblank->enabled, vblank->inmodeset);
1307 : :
1308 : : /* Drop our private "prevent drm_vblank_get" refcount */
1309 [ # # ]: 0 : if (vblank->inmodeset) {
1310 : 0 : atomic_dec(&vblank->refcount);
1311 : 0 : vblank->inmodeset = 0;
1312 : : }
1313 : :
1314 : 0 : drm_reset_vblank_timestamp(dev, pipe);
1315 : :
1316 : : /*
1317 : : * re-enable interrupts if there are users left, or the
1318 : : * user wishes vblank interrupts to be enabled all the time.
1319 : : */
1320 [ # # # # ]: 0 : if (atomic_read(&vblank->refcount) != 0 || drm_vblank_offdelay == 0)
1321 [ # # ]: 0 : WARN_ON(drm_vblank_enable(dev, pipe));
1322 : 0 : spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1323 : : }
1324 : : EXPORT_SYMBOL(drm_crtc_vblank_on);
1325 : :
1326 : : /**
1327 : : * drm_vblank_restore - estimate missed vblanks and update vblank count.
1328 : : * @dev: DRM device
1329 : : * @pipe: CRTC index
1330 : : *
1331 : : * Power manamement features can cause frame counter resets between vblank
1332 : : * disable and enable. Drivers can use this function in their
1333 : : * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1334 : : * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1335 : : * vblank counter.
1336 : : *
1337 : : * This function is the legacy version of drm_crtc_vblank_restore().
1338 : : */
1339 : 0 : void drm_vblank_restore(struct drm_device *dev, unsigned int pipe)
1340 : : {
1341 : 0 : ktime_t t_vblank;
1342 : 0 : struct drm_vblank_crtc *vblank;
1343 : 0 : int framedur_ns;
1344 : 0 : u64 diff_ns;
1345 : 0 : u32 cur_vblank, diff = 1;
1346 : 0 : int count = DRM_TIMESTAMP_MAXRETRIES;
1347 : :
1348 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1349 : 0 : return;
1350 : :
1351 [ # # ]: 0 : assert_spin_locked(&dev->vbl_lock);
1352 [ # # ]: 0 : assert_spin_locked(&dev->vblank_time_lock);
1353 : :
1354 : 0 : vblank = &dev->vblank[pipe];
1355 [ # # # # : 0 : WARN_ONCE(drm_debug_enabled(DRM_UT_VBL) && !vblank->framedur_ns,
# # # # ]
1356 : : "Cannot compute missed vblanks without frame duration\n");
1357 : 0 : framedur_ns = vblank->framedur_ns;
1358 : :
1359 : 0 : do {
1360 : 0 : cur_vblank = __get_vblank_counter(dev, pipe);
1361 : 0 : drm_get_last_vbltimestamp(dev, pipe, &t_vblank, false);
1362 [ # # # # ]: 0 : } while (cur_vblank != __get_vblank_counter(dev, pipe) && --count > 0);
1363 : :
1364 [ # # ]: 0 : diff_ns = ktime_to_ns(ktime_sub(t_vblank, vblank->time));
1365 [ # # ]: 0 : if (framedur_ns)
1366 : 0 : diff = DIV_ROUND_CLOSEST_ULL(diff_ns, framedur_ns);
1367 : :
1368 : :
1369 : 0 : DRM_DEBUG_VBL("missed %d vblanks in %lld ns, frame duration=%d ns, hw_diff=%d\n",
1370 : : diff, diff_ns, framedur_ns, cur_vblank - vblank->last);
1371 : 0 : store_vblank(dev, pipe, diff, t_vblank, cur_vblank);
1372 : : }
1373 : : EXPORT_SYMBOL(drm_vblank_restore);
1374 : :
1375 : : /**
1376 : : * drm_crtc_vblank_restore - estimate missed vblanks and update vblank count.
1377 : : * @crtc: CRTC in question
1378 : : *
1379 : : * Power manamement features can cause frame counter resets between vblank
1380 : : * disable and enable. Drivers can use this function in their
1381 : : * &drm_crtc_funcs.enable_vblank implementation to estimate missed vblanks since
1382 : : * the last &drm_crtc_funcs.disable_vblank using timestamps and update the
1383 : : * vblank counter.
1384 : : */
1385 : 0 : void drm_crtc_vblank_restore(struct drm_crtc *crtc)
1386 : : {
1387 : 0 : drm_vblank_restore(crtc->dev, drm_crtc_index(crtc));
1388 : 0 : }
1389 : : EXPORT_SYMBOL(drm_crtc_vblank_restore);
1390 : :
1391 : 0 : static void drm_legacy_vblank_pre_modeset(struct drm_device *dev,
1392 : : unsigned int pipe)
1393 : : {
1394 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1395 : :
1396 : : /* vblank is not initialized (IRQ not installed ?), or has been freed */
1397 [ # # ]: 0 : if (!dev->num_crtcs)
1398 : : return;
1399 : :
1400 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1401 : : return;
1402 : :
1403 : : /*
1404 : : * To avoid all the problems that might happen if interrupts
1405 : : * were enabled/disabled around or between these calls, we just
1406 : : * have the kernel take a reference on the CRTC (just once though
1407 : : * to avoid corrupting the count if multiple, mismatch calls occur),
1408 : : * so that interrupts remain enabled in the interim.
1409 : : */
1410 [ # # ]: 0 : if (!vblank->inmodeset) {
1411 : 0 : vblank->inmodeset = 0x1;
1412 [ # # ]: 0 : if (drm_vblank_get(dev, pipe) == 0)
1413 : 0 : vblank->inmodeset |= 0x2;
1414 : : }
1415 : : }
1416 : :
1417 : 0 : static void drm_legacy_vblank_post_modeset(struct drm_device *dev,
1418 : : unsigned int pipe)
1419 : : {
1420 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1421 : 0 : unsigned long irqflags;
1422 : :
1423 : : /* vblank is not initialized (IRQ not installed ?), or has been freed */
1424 [ # # ]: 0 : if (!dev->num_crtcs)
1425 : : return;
1426 : :
1427 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1428 : : return;
1429 : :
1430 [ # # ]: 0 : if (vblank->inmodeset) {
1431 : 0 : spin_lock_irqsave(&dev->vbl_lock, irqflags);
1432 : 0 : drm_reset_vblank_timestamp(dev, pipe);
1433 : 0 : spin_unlock_irqrestore(&dev->vbl_lock, irqflags);
1434 : :
1435 [ # # ]: 0 : if (vblank->inmodeset & 0x2)
1436 : 0 : drm_vblank_put(dev, pipe);
1437 : :
1438 : 0 : vblank->inmodeset = 0;
1439 : : }
1440 : : }
1441 : :
1442 : 0 : int drm_legacy_modeset_ctl_ioctl(struct drm_device *dev, void *data,
1443 : : struct drm_file *file_priv)
1444 : : {
1445 : 0 : struct drm_modeset_ctl *modeset = data;
1446 : 0 : unsigned int pipe;
1447 : :
1448 : : /* If drm_vblank_init() hasn't been called yet, just no-op */
1449 [ # # ]: 0 : if (!dev->num_crtcs)
1450 : : return 0;
1451 : :
1452 : : /* KMS drivers handle this internally */
1453 [ # # ]: 0 : if (!drm_core_check_feature(dev, DRIVER_LEGACY))
1454 : : return 0;
1455 : :
1456 : 0 : pipe = modeset->crtc;
1457 [ # # ]: 0 : if (pipe >= dev->num_crtcs)
1458 : : return -EINVAL;
1459 : :
1460 [ # # # ]: 0 : switch (modeset->cmd) {
1461 : 0 : case _DRM_PRE_MODESET:
1462 : 0 : drm_legacy_vblank_pre_modeset(dev, pipe);
1463 : 0 : break;
1464 : 0 : case _DRM_POST_MODESET:
1465 : 0 : drm_legacy_vblank_post_modeset(dev, pipe);
1466 : 0 : break;
1467 : : default:
1468 : : return -EINVAL;
1469 : : }
1470 : :
1471 : : return 0;
1472 : : }
1473 : :
1474 : 0 : static inline bool vblank_passed(u64 seq, u64 ref)
1475 : : {
1476 : 0 : return (seq - ref) <= (1 << 23);
1477 : : }
1478 : :
1479 : : static int drm_queue_vblank_event(struct drm_device *dev, unsigned int pipe,
1480 : : u64 req_seq,
1481 : : union drm_wait_vblank *vblwait,
1482 : : struct drm_file *file_priv)
1483 : : {
1484 : : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1485 : : struct drm_pending_vblank_event *e;
1486 : : ktime_t now;
1487 : : unsigned long flags;
1488 : : u64 seq;
1489 : : int ret;
1490 : :
1491 : : e = kzalloc(sizeof(*e), GFP_KERNEL);
1492 : : if (e == NULL) {
1493 : : ret = -ENOMEM;
1494 : : goto err_put;
1495 : : }
1496 : :
1497 : : e->pipe = pipe;
1498 : : e->event.base.type = DRM_EVENT_VBLANK;
1499 : : e->event.base.length = sizeof(e->event.vbl);
1500 : : e->event.vbl.user_data = vblwait->request.signal;
1501 : : e->event.vbl.crtc_id = 0;
1502 : : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1503 : : struct drm_crtc *crtc = drm_crtc_from_index(dev, pipe);
1504 : : if (crtc)
1505 : : e->event.vbl.crtc_id = crtc->base.id;
1506 : : }
1507 : :
1508 : : spin_lock_irqsave(&dev->event_lock, flags);
1509 : :
1510 : : /*
1511 : : * drm_crtc_vblank_off() might have been called after we called
1512 : : * drm_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1513 : : * vblank disable, so no need for further locking. The reference from
1514 : : * drm_vblank_get() protects against vblank disable from another source.
1515 : : */
1516 : : if (!READ_ONCE(vblank->enabled)) {
1517 : : ret = -EINVAL;
1518 : : goto err_unlock;
1519 : : }
1520 : :
1521 : : ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1522 : : &e->event.base);
1523 : :
1524 : : if (ret)
1525 : : goto err_unlock;
1526 : :
1527 : : seq = drm_vblank_count_and_time(dev, pipe, &now);
1528 : :
1529 : : DRM_DEBUG("event on vblank count %llu, current %llu, crtc %u\n",
1530 : : req_seq, seq, pipe);
1531 : :
1532 : : trace_drm_vblank_event_queued(file_priv, pipe, req_seq);
1533 : :
1534 : : e->sequence = req_seq;
1535 : : if (vblank_passed(seq, req_seq)) {
1536 : : drm_vblank_put(dev, pipe);
1537 : : send_vblank_event(dev, e, seq, now);
1538 : : vblwait->reply.sequence = seq;
1539 : : } else {
1540 : : /* drm_handle_vblank_events will call drm_vblank_put */
1541 : : list_add_tail(&e->base.link, &dev->vblank_event_list);
1542 : : vblwait->reply.sequence = req_seq;
1543 : : }
1544 : :
1545 : : spin_unlock_irqrestore(&dev->event_lock, flags);
1546 : :
1547 : : return 0;
1548 : :
1549 : : err_unlock:
1550 : : spin_unlock_irqrestore(&dev->event_lock, flags);
1551 : : kfree(e);
1552 : : err_put:
1553 : : drm_vblank_put(dev, pipe);
1554 : : return ret;
1555 : : }
1556 : :
1557 : 0 : static bool drm_wait_vblank_is_query(union drm_wait_vblank *vblwait)
1558 : : {
1559 : 0 : if (vblwait->request.sequence)
1560 : : return false;
1561 : :
1562 : 0 : return _DRM_VBLANK_RELATIVE ==
1563 : 0 : (vblwait->request.type & (_DRM_VBLANK_TYPES_MASK |
1564 : : _DRM_VBLANK_EVENT |
1565 : : _DRM_VBLANK_NEXTONMISS));
1566 : : }
1567 : :
1568 : : /*
1569 : : * Widen a 32-bit param to 64-bits.
1570 : : *
1571 : : * \param narrow 32-bit value (missing upper 32 bits)
1572 : : * \param near 64-bit value that should be 'close' to near
1573 : : *
1574 : : * This function returns a 64-bit value using the lower 32-bits from
1575 : : * 'narrow' and constructing the upper 32-bits so that the result is
1576 : : * as close as possible to 'near'.
1577 : : */
1578 : :
1579 : 0 : static u64 widen_32_to_64(u32 narrow, u64 near)
1580 : : {
1581 : 0 : return near + (s32) (narrow - near);
1582 : : }
1583 : :
1584 : 0 : static void drm_wait_vblank_reply(struct drm_device *dev, unsigned int pipe,
1585 : : struct drm_wait_vblank_reply *reply)
1586 : : {
1587 : 0 : ktime_t now;
1588 : 0 : struct timespec64 ts;
1589 : :
1590 : : /*
1591 : : * drm_wait_vblank_reply is a UAPI structure that uses 'long'
1592 : : * to store the seconds. This is safe as we always use monotonic
1593 : : * timestamps since linux-4.15.
1594 : : */
1595 : 0 : reply->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1596 : 0 : ts = ktime_to_timespec64(now);
1597 : 0 : reply->tval_sec = (u32)ts.tv_sec;
1598 : 0 : reply->tval_usec = ts.tv_nsec / 1000;
1599 : 0 : }
1600 : :
1601 : 0 : int drm_wait_vblank_ioctl(struct drm_device *dev, void *data,
1602 : : struct drm_file *file_priv)
1603 : : {
1604 : 0 : struct drm_crtc *crtc;
1605 : 0 : struct drm_vblank_crtc *vblank;
1606 : 0 : union drm_wait_vblank *vblwait = data;
1607 : 0 : int ret;
1608 : 0 : u64 req_seq, seq;
1609 : 0 : unsigned int pipe_index;
1610 : 0 : unsigned int flags, pipe, high_pipe;
1611 : :
1612 [ # # ]: 0 : if (!dev->irq_enabled)
1613 : : return -EOPNOTSUPP;
1614 : :
1615 [ # # ]: 0 : if (vblwait->request.type & _DRM_VBLANK_SIGNAL)
1616 : : return -EINVAL;
1617 : :
1618 [ # # ]: 0 : if (vblwait->request.type &
1619 : : ~(_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1620 : : _DRM_VBLANK_HIGH_CRTC_MASK)) {
1621 : 0 : DRM_DEBUG("Unsupported type value 0x%x, supported mask 0x%x\n",
1622 : : vblwait->request.type,
1623 : : (_DRM_VBLANK_TYPES_MASK | _DRM_VBLANK_FLAGS_MASK |
1624 : : _DRM_VBLANK_HIGH_CRTC_MASK));
1625 : 0 : return -EINVAL;
1626 : : }
1627 : :
1628 : 0 : flags = vblwait->request.type & _DRM_VBLANK_FLAGS_MASK;
1629 : 0 : high_pipe = (vblwait->request.type & _DRM_VBLANK_HIGH_CRTC_MASK);
1630 [ # # ]: 0 : if (high_pipe)
1631 : 0 : pipe_index = high_pipe >> _DRM_VBLANK_HIGH_CRTC_SHIFT;
1632 : : else
1633 : 0 : pipe_index = flags & _DRM_VBLANK_SECONDARY ? 1 : 0;
1634 : :
1635 : : /* Convert lease-relative crtc index into global crtc index */
1636 [ # # ]: 0 : if (drm_core_check_feature(dev, DRIVER_MODESET)) {
1637 : 0 : pipe = 0;
1638 [ # # ]: 0 : drm_for_each_crtc(crtc, dev) {
1639 [ # # ]: 0 : if (drm_lease_held(file_priv, crtc->base.id)) {
1640 [ # # ]: 0 : if (pipe_index == 0)
1641 : : break;
1642 : 0 : pipe_index--;
1643 : : }
1644 : 0 : pipe++;
1645 : : }
1646 : : } else {
1647 : : pipe = pipe_index;
1648 : : }
1649 : :
1650 [ # # ]: 0 : if (pipe >= dev->num_crtcs)
1651 : : return -EINVAL;
1652 : :
1653 : 0 : vblank = &dev->vblank[pipe];
1654 : :
1655 : : /* If the counter is currently enabled and accurate, short-circuit
1656 : : * queries to return the cached timestamp of the last vblank.
1657 : : */
1658 [ # # # # ]: 0 : if (dev->vblank_disable_immediate &&
1659 [ # # # # ]: 0 : drm_wait_vblank_is_query(vblwait) &&
1660 [ # # ]: 0 : READ_ONCE(vblank->enabled)) {
1661 : 0 : drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1662 : 0 : return 0;
1663 : : }
1664 : :
1665 : 0 : ret = drm_vblank_get(dev, pipe);
1666 [ # # ]: 0 : if (ret) {
1667 : 0 : DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1668 : 0 : return ret;
1669 : : }
1670 : 0 : seq = drm_vblank_count(dev, pipe);
1671 : :
1672 [ # # ]: 0 : switch (vblwait->request.type & _DRM_VBLANK_TYPES_MASK) {
1673 : 0 : case _DRM_VBLANK_RELATIVE:
1674 : 0 : req_seq = seq + vblwait->request.sequence;
1675 : 0 : vblwait->request.sequence = req_seq;
1676 : 0 : vblwait->request.type &= ~_DRM_VBLANK_RELATIVE;
1677 : 0 : break;
1678 : 0 : case _DRM_VBLANK_ABSOLUTE:
1679 : 0 : req_seq = widen_32_to_64(vblwait->request.sequence, seq);
1680 : 0 : break;
1681 : : default:
1682 : : ret = -EINVAL;
1683 : : goto done;
1684 : : }
1685 : :
1686 [ # # # # ]: 0 : if ((flags & _DRM_VBLANK_NEXTONMISS) &&
1687 : : vblank_passed(seq, req_seq)) {
1688 : 0 : req_seq = seq + 1;
1689 : 0 : vblwait->request.type &= ~_DRM_VBLANK_NEXTONMISS;
1690 : 0 : vblwait->request.sequence = req_seq;
1691 : : }
1692 : :
1693 [ # # ]: 0 : if (flags & _DRM_VBLANK_EVENT) {
1694 : : /* must hold on to the vblank ref until the event fires
1695 : : * drm_vblank_put will be called asynchronously
1696 : : */
1697 : 0 : return drm_queue_vblank_event(dev, pipe, req_seq, vblwait, file_priv);
1698 : : }
1699 : :
1700 [ # # ]: 0 : if (req_seq != seq) {
1701 : 0 : int wait;
1702 : :
1703 : 0 : DRM_DEBUG("waiting on vblank count %llu, crtc %u\n",
1704 : : req_seq, pipe);
1705 [ # # # # : 0 : wait = wait_event_interruptible_timeout(vblank->queue,
# # # # #
# # # #
# ]
1706 : : vblank_passed(drm_vblank_count(dev, pipe), req_seq) ||
1707 : : !READ_ONCE(vblank->enabled),
1708 : : msecs_to_jiffies(3000));
1709 : :
1710 [ # # # ]: 0 : switch (wait) {
1711 : : case 0:
1712 : : /* timeout */
1713 : : ret = -EBUSY;
1714 : : break;
1715 : : case -ERESTARTSYS:
1716 : : /* interrupted by signal */
1717 : 0 : ret = -EINTR;
1718 : 0 : break;
1719 : : default:
1720 : : ret = 0;
1721 : : break;
1722 : : }
1723 : 0 : }
1724 : :
1725 : 0 : if (ret != -EINTR) {
1726 : 0 : drm_wait_vblank_reply(dev, pipe, &vblwait->reply);
1727 : :
1728 : 0 : DRM_DEBUG("crtc %d returning %u to client\n",
1729 : : pipe, vblwait->reply.sequence);
1730 : : } else {
1731 : 0 : DRM_DEBUG("crtc %d vblank wait interrupted by signal\n", pipe);
1732 : : }
1733 : :
1734 : 0 : done:
1735 : 0 : drm_vblank_put(dev, pipe);
1736 : 0 : return ret;
1737 : : }
1738 : :
1739 : 0 : static void drm_handle_vblank_events(struct drm_device *dev, unsigned int pipe)
1740 : : {
1741 : 0 : struct drm_pending_vblank_event *e, *t;
1742 : 0 : ktime_t now;
1743 : 0 : u64 seq;
1744 : :
1745 [ # # ]: 0 : assert_spin_locked(&dev->event_lock);
1746 : :
1747 : 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
1748 : :
1749 [ # # ]: 0 : list_for_each_entry_safe(e, t, &dev->vblank_event_list, base.link) {
1750 [ # # ]: 0 : if (e->pipe != pipe)
1751 : 0 : continue;
1752 [ # # ]: 0 : if (!vblank_passed(seq, e->sequence))
1753 : 0 : continue;
1754 : :
1755 : 0 : DRM_DEBUG("vblank event on %llu, current %llu\n",
1756 : : e->sequence, seq);
1757 : :
1758 : 0 : list_del(&e->base.link);
1759 : 0 : drm_vblank_put(dev, pipe);
1760 : 0 : send_vblank_event(dev, e, seq, now);
1761 : : }
1762 : :
1763 : 0 : trace_drm_vblank_event(pipe, seq, now,
1764 : 0 : dev->driver->get_vblank_timestamp != NULL);
1765 : 0 : }
1766 : :
1767 : : /**
1768 : : * drm_handle_vblank - handle a vblank event
1769 : : * @dev: DRM device
1770 : : * @pipe: index of CRTC where this event occurred
1771 : : *
1772 : : * Drivers should call this routine in their vblank interrupt handlers to
1773 : : * update the vblank counter and send any signals that may be pending.
1774 : : *
1775 : : * This is the legacy version of drm_crtc_handle_vblank().
1776 : : */
1777 : 0 : bool drm_handle_vblank(struct drm_device *dev, unsigned int pipe)
1778 : : {
1779 : 0 : struct drm_vblank_crtc *vblank = &dev->vblank[pipe];
1780 : 0 : unsigned long irqflags;
1781 : 0 : bool disable_irq;
1782 : :
1783 [ # # # # ]: 0 : if (WARN_ON_ONCE(!dev->num_crtcs))
1784 : : return false;
1785 : :
1786 [ # # # # ]: 0 : if (WARN_ON(pipe >= dev->num_crtcs))
1787 : : return false;
1788 : :
1789 : 0 : spin_lock_irqsave(&dev->event_lock, irqflags);
1790 : :
1791 : : /* Need timestamp lock to prevent concurrent execution with
1792 : : * vblank enable/disable, as this would cause inconsistent
1793 : : * or corrupted timestamps and vblank counts.
1794 : : */
1795 : 0 : spin_lock(&dev->vblank_time_lock);
1796 : :
1797 : : /* Vblank irq handling disabled. Nothing to do. */
1798 [ # # ]: 0 : if (!vblank->enabled) {
1799 : 0 : spin_unlock(&dev->vblank_time_lock);
1800 : 0 : spin_unlock_irqrestore(&dev->event_lock, irqflags);
1801 : 0 : return false;
1802 : : }
1803 : :
1804 : 0 : drm_update_vblank_count(dev, pipe, true);
1805 : :
1806 : 0 : spin_unlock(&dev->vblank_time_lock);
1807 : :
1808 : 0 : wake_up(&vblank->queue);
1809 : :
1810 : : /* With instant-off, we defer disabling the interrupt until after
1811 : : * we finish processing the following vblank after all events have
1812 : : * been signaled. The disable has to be last (after
1813 : : * drm_handle_vblank_events) so that the timestamp is always accurate.
1814 : : */
1815 : 0 : disable_irq = (dev->vblank_disable_immediate &&
1816 [ # # # # : 0 : drm_vblank_offdelay > 0 &&
# # ]
1817 : 0 : !atomic_read(&vblank->refcount));
1818 : :
1819 : 0 : drm_handle_vblank_events(dev, pipe);
1820 : :
1821 : 0 : spin_unlock_irqrestore(&dev->event_lock, irqflags);
1822 : :
1823 [ # # ]: 0 : if (disable_irq)
1824 : 0 : vblank_disable_fn(&vblank->disable_timer);
1825 : :
1826 : : return true;
1827 : : }
1828 : : EXPORT_SYMBOL(drm_handle_vblank);
1829 : :
1830 : : /**
1831 : : * drm_crtc_handle_vblank - handle a vblank event
1832 : : * @crtc: where this event occurred
1833 : : *
1834 : : * Drivers should call this routine in their vblank interrupt handlers to
1835 : : * update the vblank counter and send any signals that may be pending.
1836 : : *
1837 : : * This is the native KMS version of drm_handle_vblank().
1838 : : *
1839 : : * Note that for a given vblank counter value drm_crtc_handle_vblank()
1840 : : * and drm_crtc_vblank_count() or drm_crtc_vblank_count_and_time()
1841 : : * provide a barrier: Any writes done before calling
1842 : : * drm_crtc_handle_vblank() will be visible to callers of the later
1843 : : * functions, iff the vblank count is the same or a later one.
1844 : : *
1845 : : * See also &drm_vblank_crtc.count.
1846 : : *
1847 : : * Returns:
1848 : : * True if the event was successfully handled, false on failure.
1849 : : */
1850 : 0 : bool drm_crtc_handle_vblank(struct drm_crtc *crtc)
1851 : : {
1852 : 0 : return drm_handle_vblank(crtc->dev, drm_crtc_index(crtc));
1853 : : }
1854 : : EXPORT_SYMBOL(drm_crtc_handle_vblank);
1855 : :
1856 : : /*
1857 : : * Get crtc VBLANK count.
1858 : : *
1859 : : * \param dev DRM device
1860 : : * \param data user arguement, pointing to a drm_crtc_get_sequence structure.
1861 : : * \param file_priv drm file private for the user's open file descriptor
1862 : : */
1863 : :
1864 : 0 : int drm_crtc_get_sequence_ioctl(struct drm_device *dev, void *data,
1865 : : struct drm_file *file_priv)
1866 : : {
1867 : 0 : struct drm_crtc *crtc;
1868 : 0 : struct drm_vblank_crtc *vblank;
1869 : 0 : int pipe;
1870 : 0 : struct drm_crtc_get_sequence *get_seq = data;
1871 : 0 : ktime_t now;
1872 : 0 : bool vblank_enabled;
1873 : 0 : int ret;
1874 : :
1875 [ # # ]: 0 : if (!drm_core_check_feature(dev, DRIVER_MODESET))
1876 : : return -EOPNOTSUPP;
1877 : :
1878 [ # # ]: 0 : if (!dev->irq_enabled)
1879 : : return -EOPNOTSUPP;
1880 : :
1881 : 0 : crtc = drm_crtc_find(dev, file_priv, get_seq->crtc_id);
1882 [ # # ]: 0 : if (!crtc)
1883 : : return -ENOENT;
1884 : :
1885 [ # # ]: 0 : pipe = drm_crtc_index(crtc);
1886 : :
1887 : 0 : vblank = &dev->vblank[pipe];
1888 [ # # # # ]: 0 : vblank_enabled = dev->vblank_disable_immediate && READ_ONCE(vblank->enabled);
1889 : :
1890 [ # # ]: 0 : if (!vblank_enabled) {
1891 : 0 : ret = drm_crtc_vblank_get(crtc);
1892 [ # # ]: 0 : if (ret) {
1893 : 0 : DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1894 : 0 : return ret;
1895 : : }
1896 : : }
1897 : 0 : drm_modeset_lock(&crtc->mutex, NULL);
1898 [ # # ]: 0 : if (crtc->state)
1899 : 0 : get_seq->active = crtc->state->enable;
1900 : : else
1901 : 0 : get_seq->active = crtc->enabled;
1902 : 0 : drm_modeset_unlock(&crtc->mutex);
1903 : 0 : get_seq->sequence = drm_vblank_count_and_time(dev, pipe, &now);
1904 [ # # ]: 0 : get_seq->sequence_ns = ktime_to_ns(now);
1905 [ # # ]: 0 : if (!vblank_enabled)
1906 : 0 : drm_crtc_vblank_put(crtc);
1907 : : return 0;
1908 : : }
1909 : :
1910 : : /*
1911 : : * Queue a event for VBLANK sequence
1912 : : *
1913 : : * \param dev DRM device
1914 : : * \param data user arguement, pointing to a drm_crtc_queue_sequence structure.
1915 : : * \param file_priv drm file private for the user's open file descriptor
1916 : : */
1917 : :
1918 : 0 : int drm_crtc_queue_sequence_ioctl(struct drm_device *dev, void *data,
1919 : : struct drm_file *file_priv)
1920 : : {
1921 : 0 : struct drm_crtc *crtc;
1922 : 0 : struct drm_vblank_crtc *vblank;
1923 : 0 : int pipe;
1924 : 0 : struct drm_crtc_queue_sequence *queue_seq = data;
1925 : 0 : ktime_t now;
1926 : 0 : struct drm_pending_vblank_event *e;
1927 : 0 : u32 flags;
1928 : 0 : u64 seq;
1929 : 0 : u64 req_seq;
1930 : 0 : int ret;
1931 : 0 : unsigned long spin_flags;
1932 : :
1933 [ # # ]: 0 : if (!drm_core_check_feature(dev, DRIVER_MODESET))
1934 : : return -EOPNOTSUPP;
1935 : :
1936 [ # # ]: 0 : if (!dev->irq_enabled)
1937 : : return -EOPNOTSUPP;
1938 : :
1939 : 0 : crtc = drm_crtc_find(dev, file_priv, queue_seq->crtc_id);
1940 [ # # ]: 0 : if (!crtc)
1941 : : return -ENOENT;
1942 : :
1943 : 0 : flags = queue_seq->flags;
1944 : : /* Check valid flag bits */
1945 [ # # ]: 0 : if (flags & ~(DRM_CRTC_SEQUENCE_RELATIVE|
1946 : : DRM_CRTC_SEQUENCE_NEXT_ON_MISS))
1947 : : return -EINVAL;
1948 : :
1949 : 0 : pipe = drm_crtc_index(crtc);
1950 : :
1951 : 0 : vblank = &dev->vblank[pipe];
1952 : :
1953 : 0 : e = kzalloc(sizeof(*e), GFP_KERNEL);
1954 [ # # ]: 0 : if (e == NULL)
1955 : : return -ENOMEM;
1956 : :
1957 : 0 : ret = drm_crtc_vblank_get(crtc);
1958 [ # # ]: 0 : if (ret) {
1959 : 0 : DRM_DEBUG("crtc %d failed to acquire vblank counter, %d\n", pipe, ret);
1960 : 0 : goto err_free;
1961 : : }
1962 : :
1963 : 0 : seq = drm_vblank_count_and_time(dev, pipe, &now);
1964 : 0 : req_seq = queue_seq->sequence;
1965 : :
1966 [ # # ]: 0 : if (flags & DRM_CRTC_SEQUENCE_RELATIVE)
1967 : 0 : req_seq += seq;
1968 : :
1969 [ # # # # ]: 0 : if ((flags & DRM_CRTC_SEQUENCE_NEXT_ON_MISS) && vblank_passed(seq, req_seq))
1970 : 0 : req_seq = seq + 1;
1971 : :
1972 : 0 : e->pipe = pipe;
1973 : 0 : e->event.base.type = DRM_EVENT_CRTC_SEQUENCE;
1974 : 0 : e->event.base.length = sizeof(e->event.seq);
1975 : 0 : e->event.seq.user_data = queue_seq->user_data;
1976 : :
1977 : 0 : spin_lock_irqsave(&dev->event_lock, spin_flags);
1978 : :
1979 : : /*
1980 : : * drm_crtc_vblank_off() might have been called after we called
1981 : : * drm_crtc_vblank_get(). drm_crtc_vblank_off() holds event_lock around the
1982 : : * vblank disable, so no need for further locking. The reference from
1983 : : * drm_crtc_vblank_get() protects against vblank disable from another source.
1984 : : */
1985 [ # # ]: 0 : if (!READ_ONCE(vblank->enabled)) {
1986 : 0 : ret = -EINVAL;
1987 : 0 : goto err_unlock;
1988 : : }
1989 : :
1990 : 0 : ret = drm_event_reserve_init_locked(dev, file_priv, &e->base,
1991 : : &e->event.base);
1992 : :
1993 [ # # ]: 0 : if (ret)
1994 : 0 : goto err_unlock;
1995 : :
1996 : 0 : e->sequence = req_seq;
1997 : :
1998 [ # # ]: 0 : if (vblank_passed(seq, req_seq)) {
1999 : 0 : drm_crtc_vblank_put(crtc);
2000 : 0 : send_vblank_event(dev, e, seq, now);
2001 : 0 : queue_seq->sequence = seq;
2002 : : } else {
2003 : : /* drm_handle_vblank_events will call drm_vblank_put */
2004 : 0 : list_add_tail(&e->base.link, &dev->vblank_event_list);
2005 : 0 : queue_seq->sequence = req_seq;
2006 : : }
2007 : :
2008 : 0 : spin_unlock_irqrestore(&dev->event_lock, spin_flags);
2009 : 0 : return 0;
2010 : :
2011 : 0 : err_unlock:
2012 : 0 : spin_unlock_irqrestore(&dev->event_lock, spin_flags);
2013 : 0 : drm_crtc_vblank_put(crtc);
2014 : 0 : err_free:
2015 : 0 : kfree(e);
2016 : 0 : return ret;
2017 : : }
|
