1/* $NetBSD: pmap.h,v 1.39 2016/11/11 12:06:31 maxv Exp $ */
2
3/*
4 * Copyright (c) 1997 Charles D. Cranor and Washington University.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
21 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
22 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
23 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
25 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
26 */
27
28/*
29 * Copyright (c) 2001 Wasabi Systems, Inc.
30 * All rights reserved.
31 *
32 * Written by Frank van der Linden for Wasabi Systems, Inc.
33 *
34 * Redistribution and use in source and binary forms, with or without
35 * modification, are permitted provided that the following conditions
36 * are met:
37 * 1. Redistributions of source code must retain the above copyright
38 * notice, this list of conditions and the following disclaimer.
39 * 2. Redistributions in binary form must reproduce the above copyright
40 * notice, this list of conditions and the following disclaimer in the
41 * documentation and/or other materials provided with the distribution.
42 * 3. All advertising materials mentioning features or use of this software
43 * must display the following acknowledgement:
44 * This product includes software developed for the NetBSD Project by
45 * Wasabi Systems, Inc.
46 * 4. The name of Wasabi Systems, Inc. may not be used to endorse
47 * or promote products derived from this software without specific prior
48 * written permission.
49 *
50 * THIS SOFTWARE IS PROVIDED BY WASABI SYSTEMS, INC. ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
52 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
53 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL WASABI SYSTEMS, INC
54 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
55 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
56 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
57 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
58 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
59 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
60 * POSSIBILITY OF SUCH DAMAGE.
61 */
62
63#ifndef _AMD64_PMAP_H_
64#define _AMD64_PMAP_H_
65
66#ifdef __x86_64__
67
68#if defined(_KERNEL_OPT)
69#include "opt_xen.h"
70#endif
71
72#include <sys/atomic.h>
73
74#include <machine/pte.h>
75#include <machine/segments.h>
76#ifdef _KERNEL
77#include <machine/cpufunc.h>
78#endif
79
80#include <uvm/uvm_object.h>
81#ifdef XEN
82#include <xen/xenfunc.h>
83#include <xen/xenpmap.h>
84#endif /* XEN */
85
86/*
87 * The x86_64 pmap module closely resembles the i386 one and it
88 * uses the same recursive entry scheme. See the i386 pmap.h
89 * for a description. The obvious difference is that 3 extra
90 * levels of page table need to be dealt with. The level 1 page
91 * table pages are at:
92 *
93 * l1: 0x00007f8000000000 - 0x00007fffffffffff (39 bits, needs PML4 entry)
94 *
95 * The rest is kept as physical pages in 3 UVM objects, and is
96 * temporarily mapped for virtual access when needed.
97 *
98 * Note that address space is signed, so the layout for 48 bits is:
99 *
100 * +---------------------------------+ 0xffffffffffffffff
101 * | |
102 * | Unused |
103 * | |
104 * +---------------------------------+ 0xffffff8000000000
105 * ~ ~
106 * | |
107 * | Kernel Space |
108 * | |
109 * | |
110 * +---------------------------------+ 0xffff800000000000 = 0x0000800000000000
111 * | |
112 * | alt.L1 table (PTE pages) |
113 * | |
114 * +---------------------------------+ 0x00007f8000000000
115 * ~ ~
116 * | |
117 * | User Space |
118 * | |
119 * | |
120 * +---------------------------------+ 0x0000000000000000
121 *
122 * In other words, there is a 'VA hole' at 0x0000800000000000 -
123 * 0xffff800000000000 which will trap, just as on, for example,
124 * sparcv9.
125 *
126 * The unused space can be used if needed, but it adds a little more
127 * complexity to the calculations.
128 */
129
130/*
131 * Mask to get rid of the sign-extended part of addresses.
132 */
133#define VA_SIGN_MASK 0xffff000000000000
134#define VA_SIGN_NEG(va) ((va) | VA_SIGN_MASK)
135/*
136 * XXXfvdl this one's not right.
137 */
138#define VA_SIGN_POS(va) ((va) & ~VA_SIGN_MASK)
139
140#define L4_SLOT_PTE 255
141#ifndef XEN
142#define L4_SLOT_KERN 256 /* pl4_i(VM_MIN_KERNEL_ADDRESS) */
143#else
144/* Xen use slots 256-272, let's move farther */
145#define L4_SLOT_KERN 320 /* pl4_i(VM_MIN_KERNEL_ADDRESS) */
146#endif
147#define L4_SLOT_KERNBASE 511 /* pl4_i(KERNBASE) */
148
149#define PDIR_SLOT_KERN L4_SLOT_KERN
150#define PDIR_SLOT_PTE L4_SLOT_PTE
151
152/*
153 * The following defines give the virtual addresses of various MMU
154 * data structures:
155 * PTE_BASE: the base VA of the linear PTE mappings
156 * PDP_BASE: the base VA of the recursive mapping of the PTD
157 * PDP_PDE: the VA of the PDE that points back to the PDP
158 */
159
160#define PTE_BASE ((pt_entry_t *)(L4_SLOT_PTE * NBPD_L4))
161#define KERN_BASE ((pt_entry_t *)(L4_SLOT_KERN * NBPD_L4))
162
163#define L1_BASE PTE_BASE
164#define L2_BASE ((pd_entry_t *)((char *)L1_BASE + L4_SLOT_PTE * NBPD_L3))
165#define L3_BASE ((pd_entry_t *)((char *)L2_BASE + L4_SLOT_PTE * NBPD_L2))
166#define L4_BASE ((pd_entry_t *)((char *)L3_BASE + L4_SLOT_PTE * NBPD_L1))
167
168#define PDP_PDE (L4_BASE + PDIR_SLOT_PTE)
169
170#define PDP_BASE L4_BASE
171
172#define NKL4_MAX_ENTRIES (unsigned long)1
173#define NKL3_MAX_ENTRIES (unsigned long)(NKL4_MAX_ENTRIES * 512)
174#define NKL2_MAX_ENTRIES (unsigned long)(NKL3_MAX_ENTRIES * 512)
175#define NKL1_MAX_ENTRIES (unsigned long)(NKL2_MAX_ENTRIES * 512)
176
177#define NKL4_KIMG_ENTRIES 1
178#define NKL3_KIMG_ENTRIES 1
179#define NKL2_KIMG_ENTRIES 32
180
181/*
182 * Since kva space is below the kernel in its entirety, we start off
183 * with zero entries on each level.
184 */
185#define NKL4_START_ENTRIES 0
186#define NKL3_START_ENTRIES 0
187#define NKL2_START_ENTRIES 0
188#define NKL1_START_ENTRIES 0 /* XXX */
189
190#define NTOPLEVEL_PDES (PAGE_SIZE / (sizeof (pd_entry_t)))
191
192#define NPDPG (PAGE_SIZE / sizeof (pd_entry_t))
193
194#define PTP_MASK_INITIALIZER { L1_FRAME, L2_FRAME, L3_FRAME, L4_FRAME }
195#define PTP_SHIFT_INITIALIZER { L1_SHIFT, L2_SHIFT, L3_SHIFT, L4_SHIFT }
196#define NKPTP_INITIALIZER { NKL1_START_ENTRIES, NKL2_START_ENTRIES, \
197 NKL3_START_ENTRIES, NKL4_START_ENTRIES }
198#define NKPTPMAX_INITIALIZER { NKL1_MAX_ENTRIES, NKL2_MAX_ENTRIES, \
199 NKL3_MAX_ENTRIES, NKL4_MAX_ENTRIES }
200#define NBPD_INITIALIZER { NBPD_L1, NBPD_L2, NBPD_L3, NBPD_L4 }
201#define PDES_INITIALIZER { L2_BASE, L3_BASE, L4_BASE }
202
203#define PTP_LEVELS 4
204
205/*
206 * PG_AVAIL usage: we make use of the ignored bits of the PTE
207 */
208
209#define PG_W PG_AVAIL1 /* "wired" mapping */
210#define PG_PVLIST PG_AVAIL2 /* mapping has entry on pvlist */
211/* PG_AVAIL3 not used */
212
213#define PG_X 0 /* XXX dummy */
214
215/*
216 * Number of PTE's per cache line. 8 byte pte, 64-byte cache line
217 * Used to avoid false sharing of cache lines.
218 */
219#define NPTECL 8
220
221#include <x86/pmap.h>
222
223#ifndef XEN
224#define pmap_pa2pte(a) (a)
225#define pmap_pte2pa(a) ((a) & PG_FRAME)
226#define pmap_pte_set(p, n) do { *(p) = (n); } while (0)
227#define pmap_pte_cas(p, o, n) atomic_cas_64((p), (o), (n))
228#define pmap_pte_testset(p, n) \
229 atomic_swap_ulong((volatile unsigned long *)p, n)
230#define pmap_pte_setbits(p, b) \
231 atomic_or_ulong((volatile unsigned long *)p, b)
232#define pmap_pte_clearbits(p, b) \
233 atomic_and_ulong((volatile unsigned long *)p, ~(b))
234#define pmap_pte_flush() /* nothing */
235#else
236extern kmutex_t pte_lock;
237
238static __inline pt_entry_t
239pmap_pa2pte(paddr_t pa)
240{
241 return (pt_entry_t)xpmap_ptom_masked(pa);
242}
243
244static __inline paddr_t
245pmap_pte2pa(pt_entry_t pte)
246{
247 return xpmap_mtop_masked(pte & PG_FRAME);
248}
249
250static __inline void
251pmap_pte_set(pt_entry_t *pte, pt_entry_t npte)
252{
253 int s = splvm();
254 xpq_queue_pte_update(xpmap_ptetomach(pte), npte);
255 splx(s);
256}
257
258static __inline pt_entry_t
259pmap_pte_cas(volatile pt_entry_t *ptep, pt_entry_t o, pt_entry_t n)
260{
261 pt_entry_t opte;
262
263 mutex_enter(&pte_lock);
264 opte = *ptep;
265 if (opte == o) {
266 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(ptep)), n);
267 xpq_flush_queue();
268 }
269
270 mutex_exit(&pte_lock);
271 return opte;
272}
273
274static __inline pt_entry_t
275pmap_pte_testset(volatile pt_entry_t *pte, pt_entry_t npte)
276{
277 pt_entry_t opte;
278
279 mutex_enter(&pte_lock);
280 opte = *pte;
281 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), npte);
282 xpq_flush_queue();
283 mutex_exit(&pte_lock);
284 return opte;
285}
286
287static __inline void
288pmap_pte_setbits(volatile pt_entry_t *pte, pt_entry_t bits)
289{
290 mutex_enter(&pte_lock);
291 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)), (*pte) | bits);
292 xpq_flush_queue();
293 mutex_exit(&pte_lock);
294}
295
296static __inline void
297pmap_pte_clearbits(volatile pt_entry_t *pte, pt_entry_t bits)
298{
299 mutex_enter(&pte_lock);
300 xpq_queue_pte_update(xpmap_ptetomach(__UNVOLATILE(pte)),
301 (*pte) & ~bits);
302 xpq_flush_queue();
303 mutex_exit(&pte_lock);
304}
305
306static __inline void
307pmap_pte_flush(void)
308{
309 int s = splvm();
310 xpq_flush_queue();
311 splx(s);
312}
313#endif
314
315void pmap_changeprot_local(vaddr_t, vm_prot_t);
316
317#include <x86/pmap_pv.h>
318
319#define __HAVE_VM_PAGE_MD
320#define VM_MDPAGE_INIT(pg) \
321 memset(&(pg)->mdpage, 0, sizeof((pg)->mdpage)); \
322 PMAP_PAGE_INIT(&(pg)->mdpage.mp_pp)
323
324struct vm_page_md {
325 struct pmap_page mp_pp;
326};
327
328#else /* !__x86_64__ */
329
330#include <i386/pmap.h>
331
332#endif /* __x86_64__ */
333
334#endif /* _AMD64_PMAP_H_ */
335