changelog shortlog tags branches changeset files revisions annotate raw help

Mercurial > hg > plan9front / sys/src/9/bcm64/mmu.c

changeset 7387: 27f7b5d5b78a
parent: fffb0beda81a
author: cinap_lenrek@felloff.net
date: Sat, 14 Sep 2019 14:02:34 +0200
permissions: -rw-r--r--
description: bcm64: enter page tables in mmutop *AFTER* switching asid in mmuswitch()

there was a small window between modifying mmutop and switching the
asid where the core could bring in the new entries under the old asid
into the tlb due to speculation / prefetching.

this change moves the entering of the page tables into mmutop after
setttbr() to prevent this scenario.

due to us switching to the resereved asid 0 on procsave()->putasid(),
the only asid that could have potentially been poisoned would be asid 0
which does not have any user mappings. so this did not show any noticable
effect.
1 #include "u.h"
2 #include "../port/lib.h"
3 #include "mem.h"
4 #include "dat.h"
5 #include "fns.h"
6 #include "sysreg.h"
7 
8 void
9 mmu0init(uintptr *l1)
10 {
11  uintptr va, pa, pe, attr;
12 
13  /* KZERO */
14  attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTESH(SHARE_INNER);
15  pe = -KZERO;
16  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
17  l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
18  l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | attr;
19  }
20 
21  /* VIRTIO */
22  attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTEPXN | PTESH(SHARE_OUTER) | PTEDEVICE;
23  pe = soc.physio + soc.iosize;
24  for(pa = soc.physio, va = soc.virtio; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
25  if(((pa|va) & PGLSZ(1)-1) != 0){
26  l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
27  for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
28  assert(l1[PTLX(va, 0)] == 0);
29  l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
30  }
31  break;
32  }
33  l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
34  }
35 
36  /* ARMLOCAL */
37  attr = PTEWRITE | PTEAF | PTEKERNEL | PTEUXN | PTEPXN | PTESH(SHARE_OUTER) | PTEDEVICE;
38  pe = soc.armlocal + MB;
39  for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
40  if(((pa|va) & PGLSZ(1)-1) != 0){
41  l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
42  for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
43  assert(l1[PTLX(va, 0)] == 0);
44  l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
45  }
46  break;
47  }
48  l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
49  }
50 
51  if(PTLEVELS > 2)
52  for(va = KSEG0; va != 0; va += PGLSZ(2))
53  l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
54  if(PTLEVELS > 3)
55  for(va = KSEG0; va != 0; va += PGLSZ(3))
56  l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
57 }
58 
59 void
60 mmu0clear(uintptr *l1)
61 {
62  uintptr va, pa, pe;
63 
64  pe = -KZERO;
65  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
66  if(PTL1X(pa, 1) != PTL1X(va, 1))
67  l1[PTL1X(pa, 1)] = 0;
68  if(PTLEVELS > 2)
69  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
70  if(PTL1X(pa, 2) != PTL1X(va, 2))
71  l1[PTL1X(pa, 2)] = 0;
72  if(PTLEVELS > 3)
73  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
74  if(PTL1X(pa, 3) != PTL1X(va, 3))
75  l1[PTL1X(pa, 3)] = 0;
76 }
77 
78 void
79 mmuidmap(uintptr *l1)
80 {
81  uintptr va, pa, pe;
82 
83  pe = PHYSDRAM + soc.dramsize;
84  if(pe > (uintptr)-KZERO)
85  pe = (uintptr)-KZERO;
86  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
87  l1[PTL1X(pa, 1)] = l1[PTL1X(va, 1)];
88  if(PTLEVELS > 2)
89  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
90  l1[PTL1X(pa, 2)] = l1[PTL1X(va, 2)];
91  if(PTLEVELS > 3)
92  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
93  l1[PTL1X(pa, 3)] = l1[PTL1X(va, 3)];
94  setttbr(PADDR(&l1[L1TABLEX(0, PTLEVELS-1)]));
95  flushtlb();
96 }
97 
98 void
99 mmu1init(void)
100 {
101  m->mmutop = mallocalign(L1TOPSIZE, BY2PG, 0, 0);
102  if(m->mmutop == nil)
103  panic("mmu1init: no memory for mmutop");
104  memset(m->mmutop, 0, L1TOPSIZE);
105  mmuswitch(nil);
106 }
107 
108 /* KZERO maps the first 1GB of ram */
109 uintptr
110 paddr(void *va)
111 {
112  if((uintptr)va >= KZERO)
113  return (uintptr)va-KZERO;
114  panic("paddr: va=%#p pc=%#p", va, getcallerpc(&va));
115  return 0;
116 }
117 
118 uintptr
119 cankaddr(uintptr pa)
120 {
121  if(pa < (uintptr)-KZERO)
122  return -KZERO - pa;
123  return 0;
124 }
125 
126 void*
127 kaddr(uintptr pa)
128 {
129  if(pa < (uintptr)-KZERO)
130  return (void*)(pa + KZERO);
131  panic("kaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
132  return nil;
133 }
134 
135 /* KMAP maps all of ram (up to 4GB) */
136 static void*
137 kmapaddr(uintptr pa)
138 {
139  if(pa < (uintptr)-KZERO)
140  return (void*)(pa + KZERO);
141  return (void*)(pa + KMAP);
142 }
143 
144 KMap*
145 kmap(Page *p)
146 {
147  return kmapaddr(p->pa);
148 }
149 
150 void
151 kunmap(KMap*)
152 {
153 }
154 
155 void
156 kmapinval(void)
157 {
158 }
159 
160 #define INITMAP (ROUND((uintptr)end + BY2PG, PGLSZ(1))-KZERO)
161 
162 static void*
163 rampage(void)
164 {
165  uintptr pa;
166 
167  if(conf.npage)
168  return mallocalign(BY2PG, BY2PG, 0, 0);
169 
170  pa = conf.mem[0].base;
171  assert((pa % BY2PG) == 0);
172  assert(pa < INITMAP);
173  conf.mem[0].base += BY2PG;
174  return KADDR(pa);
175 }
176 
177 static void
178 l1map(uintptr va, uintptr pa, uintptr pe, uintptr attr)
179 {
180  uintptr *l1, *l0;
181 
182  assert(pa < pe);
183 
184  va &= -BY2PG;
185  pa &= -BY2PG;
186  pe = PGROUND(pe);
187 
188  attr |= PTEKERNEL | PTEAF;
189 
190  l1 = (uintptr*)L1;
191 
192  while(pa < pe){
193  if(l1[PTL1X(va, 1)] == 0 && (pe-pa) >= PGLSZ(1) && ((va|pa) & PGLSZ(1)-1) == 0){
194  l1[PTL1X(va, 1)] = PTEVALID | PTEBLOCK | pa | attr;
195  va += PGLSZ(1);
196  pa += PGLSZ(1);
197  continue;
198  }
199  if(l1[PTL1X(va, 1)] & PTEVALID) {
200  assert((l1[PTL1X(va, 1)] & PTETABLE) == PTETABLE);
201  l0 = KADDR(l1[PTL1X(va, 1)] & -PGLSZ(0));
202  } else {
203  l0 = rampage();
204  memset(l0, 0, BY2PG);
205  l1[PTL1X(va, 1)] = PTEVALID | PTETABLE | PADDR(l0);
206  }
207  assert(l0[PTLX(va, 0)] == 0);
208  l0[PTLX(va, 0)] = PTEVALID | PTEPAGE | pa | attr;
209  va += BY2PG;
210  pa += BY2PG;
211  }
212 }
213 
214 static void
215 kmapram(uintptr base, uintptr limit)
216 {
217  if(base < (uintptr)-KZERO && limit > (uintptr)-KZERO){
218  kmapram(base, (uintptr)-KZERO);
219  kmapram((uintptr)-KZERO, limit);
220  return;
221  }
222  if(base < INITMAP)
223  base = INITMAP;
224  if(base >= limit || limit <= INITMAP)
225  return;
226 
227  l1map((uintptr)kmapaddr(base), base, limit,
228  PTEWRITE | PTEPXN | PTEUXN | PTESH(SHARE_INNER));
229 }
230 
231 void
232 meminit(void)
233 {
234  uvlong memsize = 0;
235  uintptr pa, va;
236  char *p, *e;
237  int i;
238 
239  if(p = getconf("*maxmem")){
240  memsize = strtoull(p, &e, 0) - PHYSDRAM;
241  for(i = 1; i < nelem(conf.mem); i++){
242  if(e <= p || *e != ' ')
243  break;
244  p = ++e;
245  conf.mem[i].base = strtoull(p, &e, 0);
246  if(e <= p || *e != ' ')
247  break;
248  p = ++e;
249  conf.mem[i].limit = strtoull(p, &e, 0);
250  }
251  }
252 
253  if (memsize < INITMAP) /* sanity */
254  memsize = INITMAP;
255 
256  getramsize(&conf.mem[0]);
257  if(conf.mem[0].limit == 0){
258  conf.mem[0].base = PHYSDRAM;
259  conf.mem[0].limit = PHYSDRAM + memsize;
260  }else if(p != nil)
261  conf.mem[0].limit = conf.mem[0].base + memsize;
262 
263  /*
264  * now we know the real memory regions, unmap
265  * everything above INITMAP and map again with
266  * the proper sizes.
267  */
268  coherence();
269  for(va = INITMAP+KZERO; va != 0; va += PGLSZ(1)){
270  pa = va-KZERO;
271  ((uintptr*)L1)[PTL1X(pa, 1)] = 0;
272  ((uintptr*)L1)[PTL1X(va, 1)] = 0;
273  }
274  flushtlb();
275 
276  pa = PGROUND((uintptr)end)-KZERO;
277  for(i=0; i<nelem(conf.mem); i++){
278  if(conf.mem[i].limit <= conf.mem[i].base
279  || conf.mem[i].base >= PHYSDRAM + soc.dramsize){
280  conf.mem[i].base = conf.mem[i].limit = 0;
281  continue;
282  }
283  if(conf.mem[i].limit > PHYSDRAM + soc.dramsize)
284  conf.mem[i].limit = PHYSDRAM + soc.dramsize;
285 
286  /* take kernel out of allocatable space */
287  if(pa > conf.mem[i].base && pa < conf.mem[i].limit)
288  conf.mem[i].base = pa;
289 
290  kmapram(conf.mem[i].base, conf.mem[i].limit);
291  }
292  flushtlb();
293 
294  /* rampage() is now done, count up the pages for each bank */
295  for(i=0; i<nelem(conf.mem); i++)
296  conf.mem[i].npage = (conf.mem[i].limit - conf.mem[i].base)/BY2PG;
297 }
298 
299 uintptr
300 mmukmap(uintptr va, uintptr pa, usize size)
301 {
302  uintptr attr, off;
303 
304  if(va == 0)
305  return 0;
306 
307  off = pa & BY2PG-1;
308 
309  attr = va & PTEMA(7);
310  attr |= PTEWRITE | PTEUXN | PTEPXN | PTESH(SHARE_OUTER);
311 
312  va &= -BY2PG;
313  pa &= -BY2PG;
314 
315  l1map(va, pa, pa + off + size, attr);
316  flushtlb();
317 
318  return va + off;
319 }
320 
321 void*
322 vmap(uintptr pa, int size)
323 {
324  static uintptr base = VMAP;
325  uintptr pe = pa + size;
326  uintptr va;
327 
328  va = base;
329  base += PGROUND(pe) - (pa & -BY2PG);
330 
331  return (void*)mmukmap(va | PTEDEVICE, pa, size);
332 }
333 
334 void
335 vunmap(void *, int)
336 {
337 }
338 
339 static uintptr*
340 mmuwalk(uintptr va, int level)
341 {
342  uintptr *table, pte;
343  Page *pg;
344  int i, x;
345 
346  x = PTLX(va, PTLEVELS-1);
347  table = m->mmutop;
348  for(i = PTLEVELS-2; i >= level; i--){
349  pte = table[x];
350  if(pte & PTEVALID) {
351  if(pte & (0xFFFFULL<<48))
352  iprint("strange pte %#p va %#p\n", pte, va);
353  pte &= ~(0xFFFFULL<<48 | BY2PG-1);
354  } else {
355  pg = up->mmufree;
356  if(pg == nil)
357  return nil;
358  up->mmufree = pg->next;
359  pg->va = va & -PGLSZ(i+1);
360  if((pg->next = up->mmuhead[i+1]) == nil)
361  up->mmutail[i+1] = pg;
362  up->mmuhead[i+1] = pg;
363  pte = pg->pa;
364  memset(kmapaddr(pte), 0, BY2PG);
365  coherence();
366  table[x] = pte | PTEVALID | PTETABLE;
367  }
368  table = kmapaddr(pte);
369  x = PTLX(va, (uintptr)i);
370  }
371  return &table[x];
372 }
373 
374 static Proc *asidlist[256];
375 
376 static int
377 allocasid(Proc *p)
378 {
379  static Lock lk;
380  Proc *x;
381  int a;
382 
383  lock(&lk);
384  a = p->asid;
385  if(a < 0)
386  a = -a;
387  if(a == 0)
388  a = p->pid;
389  for(;; a++){
390  a %= nelem(asidlist);
391  if(a == 0)
392  continue; // reserved
393  x = asidlist[a];
394  if(x == p || x == nil || (x->asid < 0 && x->mach == nil))
395  break;
396  }
397  p->asid = a;
398  asidlist[a] = p;
399  unlock(&lk);
400 
401  return x != p;
402 }
403 
404 static void
405 freeasid(Proc *p)
406 {
407  int a;
408 
409  a = p->asid;
410  if(a < 0)
411  a = -a;
412  if(a > 0 && asidlist[a] == p)
413  asidlist[a] = nil;
414  p->asid = 0;
415 }
416 
417 void
418 putasid(Proc *p)
419 {
420  /*
421  * Prevent the following scenario:
422  * pX sleeps on cpuA, leaving its page tables in mmutop
423  * pX wakes up on cpuB, and exits, freeing its page tables
424  * pY on cpuB allocates a freed page table page and overwrites with data
425  * cpuA takes an interrupt, and is now running with bad page tables
426  * In theory this shouldn't hurt because only user address space tables
427  * are affected, and mmuswitch will clear mmutop before a user process is
428  * dispatched. But empirically it correlates with weird problems, eg
429  * resetting of the core clock at 0x4000001C which confuses local timers.
430  */
431  if(conf.nmach > 1)
432  mmuswitch(nil);
433 
434  if(p->asid > 0)
435  p->asid = -p->asid;
436 }
437 
438 void
439 putmmu(uintptr va, uintptr pa, Page *pg)
440 {
441  uintptr *pte, old;
442  int s;
443 
444  s = splhi();
445  while((pte = mmuwalk(va, 0)) == nil){
446  spllo();
447  up->mmufree = newpage(0, nil, 0);
448  splhi();
449  }
450  old = *pte;
451  *pte = 0;
452  if((old & PTEVALID) != 0)
453  flushasidvall((uvlong)up->asid<<48 | va>>12);
454  else
455  flushasidva((uvlong)up->asid<<48 | va>>12);
456  *pte = pa | PTEPAGE | PTEUSER | PTEPXN | PTENG | PTEAF |
457  (((pa & PTEMA(7)) == PTECACHED)? PTESH(SHARE_INNER): PTESH(SHARE_OUTER));
458  if(pg->txtflush & (1UL<<m->machno)){
459  /* pio() sets PG_TXTFLUSH whenever a text pg has been written */
460  cachedwbinvse(kmap(pg), BY2PG);
461  cacheiinvse((void*)va, BY2PG);
462  pg->txtflush &= ~(1UL<<m->machno);
463  }
464  splx(s);
465 }
466 
467 static void
468 mmufree(Proc *p)
469 {
470  int i;
471 
472  freeasid(p);
473 
474  for(i=1; i<PTLEVELS; i++){
475  if(p->mmuhead[i] == nil)
476  break;
477  p->mmutail[i]->next = p->mmufree;
478  p->mmufree = p->mmuhead[i];
479  p->mmuhead[i] = p->mmutail[i] = nil;
480  }
481 }
482 
483 void
484 mmuswitch(Proc *p)
485 {
486  uintptr va;
487  Page *t;
488 
489  for(va = UZERO; va < USTKTOP; va += PGLSZ(PTLEVELS-1))
490  m->mmutop[PTLX(va, PTLEVELS-1)] = 0;
491 
492  if(p == nil){
493  setttbr(PADDR(m->mmutop));
494  return;
495  }
496 
497  if(p->newtlb){
498  mmufree(p);
499  p->newtlb = 0;
500  }
501 
502  if(allocasid(p))
503  flushasid((uvlong)p->asid<<48);
504 
505  setttbr((uvlong)p->asid<<48 | PADDR(m->mmutop));
506 
507  for(t = p->mmuhead[PTLEVELS-1]; t != nil; t = t->next){
508  va = t->va;
509  m->mmutop[PTLX(va, PTLEVELS-1)] = t->pa | PTEVALID | PTETABLE;
510  }
511 }
512 
513 void
514 mmurelease(Proc *p)
515 {
516  Page *t;
517 
518  mmuswitch(nil);
519  mmufree(p);
520 
521  if((t = p->mmufree) != nil){
522  do {
523  p->mmufree = t->next;
524  if(--t->ref != 0)
525  panic("mmurelease: bad page ref");
526  pagechainhead(t);
527  } while((t = p->mmufree) != nil);
528  pagechaindone();
529  }
530 }
531 
532 void
533 flushmmu(void)
534 {
535  int x;
536 
537  x = splhi();
538  up->newtlb = 1;
539  mmuswitch(up);
540  splx(x);
541 }
542 
543 void
544 checkmmu(uintptr, uintptr)
545 {
546 }