Branch data Line data Source code
1 : : /*
2 : : * Copyright © 2014 Intel Corporation
3 : : *
4 : : * Permission is hereby granted, free of charge, to any person obtaining a
5 : : * copy of this software and associated documentation files (the "Software"),
6 : : * to deal in the Software without restriction, including without limitation
7 : : * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 : : * and/or sell copies of the Software, and to permit persons to whom the
9 : : * Software is furnished to do so, subject to the following conditions:
10 : : *
11 : : * The above copyright notice and this permission notice (including the next
12 : : * paragraph) shall be included in all copies or substantial portions of the
13 : : * Software.
14 : : *
15 : : * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 : : * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 : : * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 : : * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 : : * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 : : * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 : : * IN THE SOFTWARE.
22 : : *
23 : : */
24 : :
25 : : #include <linux/mm.h>
26 : : #include <linux/io-mapping.h>
27 : :
28 : : #include <asm/pgtable.h>
29 : :
30 : : #include "i915_drv.h"
31 : :
32 : : struct remap_pfn {
33 : : struct mm_struct *mm;
34 : : unsigned long pfn;
35 : : pgprot_t prot;
36 : :
37 : : struct sgt_iter sgt;
38 : : resource_size_t iobase;
39 : : };
40 : :
41 : 0 : static int remap_pfn(pte_t *pte, unsigned long addr, void *data)
42 : : {
43 : 0 : struct remap_pfn *r = data;
44 : :
45 : : /* Special PTE are not associated with any struct page */
46 [ # # ]: 0 : set_pte_at(r->mm, addr, pte, pte_mkspecial(pfn_pte(r->pfn, r->prot)));
47 : 0 : r->pfn++;
48 : :
49 : 0 : return 0;
50 : : }
51 : :
52 : : #define use_dma(io) ((io) != -1)
53 : :
54 : 0 : static inline unsigned long sgt_pfn(const struct remap_pfn *r)
55 : : {
56 : 0 : if (use_dma(r->iobase))
57 : 0 : return (r->sgt.dma + r->sgt.curr + r->iobase) >> PAGE_SHIFT;
58 : : else
59 : 0 : return r->sgt.pfn + (r->sgt.curr >> PAGE_SHIFT);
60 : : }
61 : :
62 : 0 : static int remap_sg(pte_t *pte, unsigned long addr, void *data)
63 : : {
64 : 0 : struct remap_pfn *r = data;
65 : :
66 [ # # ]: 0 : if (GEM_WARN_ON(!r->sgt.pfn))
67 : : return -EINVAL;
68 : :
69 : : /* Special PTE are not associated with any struct page */
70 [ # # ]: 0 : set_pte_at(r->mm, addr, pte,
71 : : pte_mkspecial(pfn_pte(sgt_pfn(r), r->prot)));
72 : 0 : r->pfn++; /* track insertions in case we need to unwind later */
73 : :
74 : 0 : r->sgt.curr += PAGE_SIZE;
75 [ # # ]: 0 : if (r->sgt.curr >= r->sgt.max)
76 [ # # ]: 0 : r->sgt = __sgt_iter(__sg_next(r->sgt.sgp), use_dma(r->iobase));
77 : :
78 : : return 0;
79 : : }
80 : :
81 : : /**
82 : : * remap_io_mapping - remap an IO mapping to userspace
83 : : * @vma: user vma to map to
84 : : * @addr: target user address to start at
85 : : * @pfn: physical address of kernel memory
86 : : * @size: size of map area
87 : : * @iomap: the source io_mapping
88 : : *
89 : : * Note: this is only safe if the mm semaphore is held when called.
90 : : */
91 : 0 : int remap_io_mapping(struct vm_area_struct *vma,
92 : : unsigned long addr, unsigned long pfn, unsigned long size,
93 : : struct io_mapping *iomap)
94 : : {
95 : 0 : struct remap_pfn r;
96 : 0 : int err;
97 : :
98 : : #define EXPECTED_FLAGS (VM_PFNMAP | VM_DONTEXPAND | VM_DONTDUMP)
99 : 0 : GEM_BUG_ON((vma->vm_flags & EXPECTED_FLAGS) != EXPECTED_FLAGS);
100 : :
101 : : /* We rely on prevalidation of the io-mapping to skip track_pfn(). */
102 : 0 : r.mm = vma->vm_mm;
103 : 0 : r.pfn = pfn;
104 : 0 : r.prot = __pgprot((pgprot_val(iomap->prot) & _PAGE_CACHE_MASK) |
105 : : (pgprot_val(vma->vm_page_prot) & ~_PAGE_CACHE_MASK));
106 : :
107 : 0 : err = apply_to_page_range(r.mm, addr, size, remap_pfn, &r);
108 [ # # ]: 0 : if (unlikely(err)) {
109 : 0 : zap_vma_ptes(vma, addr, (r.pfn - pfn) << PAGE_SHIFT);
110 : 0 : return err;
111 : : }
112 : :
113 : : return 0;
114 : : }
115 : :
116 : : /**
117 : : * remap_io_sg - remap an IO mapping to userspace
118 : : * @vma: user vma to map to
119 : : * @addr: target user address to start at
120 : : * @size: size of map area
121 : : * @sgl: Start sg entry
122 : : * @iobase: Use stored dma address offset by this address or pfn if -1
123 : : *
124 : : * Note: this is only safe if the mm semaphore is held when called.
125 : : */
126 : 0 : int remap_io_sg(struct vm_area_struct *vma,
127 : : unsigned long addr, unsigned long size,
128 : : struct scatterlist *sgl, resource_size_t iobase)
129 : : {
130 : 0 : struct remap_pfn r = {
131 [ # # ]: 0 : .mm = vma->vm_mm,
132 : : .prot = vma->vm_page_prot,
133 : : .sgt = __sgt_iter(sgl, use_dma(iobase)),
134 : : .iobase = iobase,
135 : : };
136 : 0 : int err;
137 : :
138 : : /* We rely on prevalidation of the io-mapping to skip track_pfn(). */
139 : 0 : GEM_BUG_ON((vma->vm_flags & EXPECTED_FLAGS) != EXPECTED_FLAGS);
140 : :
141 : 0 : if (!use_dma(iobase))
142 : : flush_cache_range(vma, addr, size);
143 : :
144 : 0 : err = apply_to_page_range(r.mm, addr, size, remap_sg, &r);
145 [ # # ]: 0 : if (unlikely(err)) {
146 : 0 : zap_vma_ptes(vma, addr, r.pfn << PAGE_SHIFT);
147 : 0 : return err;
148 : : }
149 : :
150 : : return 0;
151 : : }
|
