changelog shortlog tags branches changeset files revisions annotate raw help

Mercurial > hg > plan9front / sys/src/9/bcm64/mmu.c

changeset 7235: b1dc95374307
parent: 871931727b28
child: 9fe2319844b6
author: cinap_lenrek@felloff.net
date: Mon, 13 May 2019 19:20:21 +0200
permissions: -rw-r--r--
description: bcm64: implement reboot support
1 #include "u.h"
2 #include "../port/lib.h"
3 #include "mem.h"
4 #include "dat.h"
5 #include "fns.h"
6 #include "sysreg.h"
7 
8 void
9 mmu0init(uintptr *l1)
10 {
11  uintptr va, pa, pe;
12 
13  /* 0 identity map */
14  pe = PHYSDRAM + soc.dramsize;
15  for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(1))
16  l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
17  | PTEKERNEL | PTESH(SHARE_INNER);
18  if(PTLEVELS > 2)
19  for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(2))
20  l1[PTL1X(pa, 2)] = (uintptr)&l1[L1TABLEX(pa, 1)] | PTEVALID | PTETABLE;
21  if(PTLEVELS > 3)
22  for(pa = PHYSDRAM; pa < pe; pa += PGLSZ(3))
23  l1[PTL1X(pa, 3)] = (uintptr)&l1[L1TABLEX(pa, 2)] | PTEVALID | PTETABLE;
24 
25  /* KZERO */
26  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
27  l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
28  | PTEKERNEL | PTESH(SHARE_INNER);
29  if(PTLEVELS > 2)
30  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
31  l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
32  if(PTLEVELS > 3)
33  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
34  l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
35 
36  /* VIRTIO */
37  pe = -VIRTIO + soc.physio;
38  for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
39  l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
40  | PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
41  if(PTLEVELS > 2)
42  for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
43  l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
44  if(PTLEVELS > 3)
45  for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
46  l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
47 }
48 
49 void
50 mmu0clear(uintptr *l1)
51 {
52  uintptr va, pa, pe;
53 
54  pe = PHYSDRAM + soc.dramsize;
55 
56  if(PTLEVELS > 3)
57  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
58  if(PTL1X(pa, 1) != PTL1X(va, 1))
59  l1[PTL1X(pa, 1)] = 0;
60  }
61  if(PTLEVELS > 2)
62  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2)){
63  if(PTL1X(pa, 2) != PTL1X(va, 2))
64  l1[PTL1X(pa, 2)] = 0;
65  }
66  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3)){
67  if(PTL1X(pa, 3) != PTL1X(va, 3))
68  l1[PTL1X(pa, 3)] = 0;
69  }
70 }
71 
72 void
73 mmuidmap(uintptr *l1)
74 {
75  uintptr va, pa, pe;
76 
77  mmuswitch(nil);
78  flushtlb();
79 
80  pe = PHYSDRAM + soc.dramsize;
81 
82  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
83  if(PTL1X(pa, 1) != PTL1X(va, 1))
84  l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
85  | PTEKERNEL | PTESH(SHARE_INNER);
86  }
87  if(PTLEVELS > 2)
88  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2)){
89  if(PTL1X(pa, 2) != PTL1X(va, 2))
90  l1[PTL1X(pa, 2)] = PADDR(&l1[L1TABLEX(pa, 1)]) | PTEVALID | PTETABLE;
91  }
92  if(PTLEVELS > 3)
93  for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3)){
94  if(PTL1X(pa, 3) != PTL1X(va, 3))
95  l1[PTL1X(pa, 3)] = PADDR(&l1[L1TABLEX(pa, 2)]) | PTEVALID | PTETABLE;
96  }
97  setttbr(PADDR(&l1[L1TABLEX(0, PTLEVELS-1)]));
98 }
99 
100 void
101 mmu1init(void)
102 {
103  m->mmul1 = mallocalign(L1SIZE+L1TOPSIZE, BY2PG, L1SIZE, 0);
104  if(m->mmul1 == nil)
105  panic("mmu1init: no memory for mmul1");
106  memset(m->mmul1, 0, L1SIZE+L1TOPSIZE);
107  mmuswitch(nil);
108 }
109 
110 uintptr
111 paddr(void *va)
112 {
113  if((uintptr)va >= KZERO)
114  return (uintptr)va-KZERO;
115  panic("paddr: va=%#p pc=%#p", va, getcallerpc(&va));
116  return 0;
117 }
118 
119 uintptr
120 cankaddr(uintptr pa)
121 {
122  if(pa < (uintptr)-KZERO)
123  return -KZERO - pa;
124  return 0;
125 }
126 
127 void*
128 kaddr(uintptr pa)
129 {
130  if(pa < (uintptr)-KZERO)
131  return (void*)(pa + KZERO);
132  panic("kaddr: pa=%#p pc=%#p", pa, getcallerpc(&pa));
133  return nil;
134 }
135 
136 void
137 kmapinval(void)
138 {
139 }
140 
141 KMap*
142 kmap(Page *p)
143 {
144  return kaddr(p->pa);
145 }
146 
147 void
148 kunmap(KMap*)
149 {
150 }
151 
152 uintptr
153 mmukmap(uintptr va, uintptr pa, usize size)
154 {
155  uintptr a, pe, off, attr;
156 
157  if(va == 0)
158  return 0;
159 
160  attr = va & PTEMA(7);
161  va &= -PGLSZ(1);
162  off = pa % PGLSZ(1);
163  a = va + off;
164  pe = (pa + size + (PGLSZ(1)-1)) & -PGLSZ(1);
165  while(pa < pe){
166  ((uintptr*)L1)[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | PTEWRITE | PTEAF
167  | PTEKERNEL | PTESH(SHARE_OUTER) | attr;
168  pa += PGLSZ(1);
169  va += PGLSZ(1);
170  }
171  flushtlb();
172  return a;
173 }
174 
175 static uintptr*
176 mmuwalk(uintptr va, int level)
177 {
178  uintptr *table, pte;
179  Page *pg;
180  int i, x;
181 
182  x = PTLX(va, PTLEVELS-1);
183  table = &m->mmul1[L1TABLEX(va, PTLEVELS-1)];
184  for(i = PTLEVELS-2; i >= level; i--){
185  pte = table[x];
186  if(pte & PTEVALID) {
187  if(pte & (0xFFFFULL<<48))
188  iprint("strange pte %#p va %#p\n", pte, va);
189  pte &= ~(0xFFFFULL<<48 | BY2PG-1);
190  table = KADDR(pte);
191  } else {
192  if(i < 2){
193  pg = up->mmufree;
194  if(pg == nil)
195  return nil;
196  up->mmufree = pg->next;
197  switch(i){
198  case 0:
199  pg->va = va & -PGLSZ(1);
200  if((pg->next = up->mmul1) == nil)
201  up->mmul1tail = pg;
202  up->mmul1 = pg;
203  break;
204  case 1:
205  pg->va = va & -PGLSZ(2);
206  if((pg->next = up->mmul2) == nil)
207  up->mmul2tail = pg;
208  up->mmul2 = pg;
209  break;
210  }
211  memset(KADDR(pg->pa), 0, BY2PG);
212  coherence();
213  table[x] = pg->pa | PTEVALID | PTETABLE;
214  table = KADDR(pg->pa);
215  } else {
216  table[x] = PADDR(&m->mmul1[L1TABLEX(va, 2)]) | PTEVALID | PTETABLE;
217  table = &m->mmul1[L1TABLEX(va, 2)];
218  }
219  }
220  x = PTLX(va, (uintptr)i);
221  }
222  return &table[x];
223 }
224 
225 static Proc *asidlist[256];
226 
227 static int
228 allocasid(Proc *p)
229 {
230  static Lock lk;
231  Proc *x;
232  int a;
233 
234  lock(&lk);
235  a = p->asid;
236  if(a < 0)
237  a = -a;
238  if(a == 0)
239  a = p->pid;
240  for(;; a++){
241  a %= nelem(asidlist);
242  if(a == 0)
243  continue; // reserved
244  x = asidlist[a];
245  if(x == p || x == nil || (x->asid < 0 && x->mach == nil))
246  break;
247  }
248  p->asid = a;
249  asidlist[a] = p;
250  unlock(&lk);
251 
252  return x != p;
253 }
254 
255 static void
256 freeasid(Proc *p)
257 {
258  int a;
259 
260  a = p->asid;
261  if(a < 0)
262  a = -a;
263  if(a > 0 && asidlist[a] == p)
264  asidlist[a] = nil;
265  p->asid = 0;
266 }
267 
268 void
269 putasid(Proc *p)
270 {
271  /*
272  * Prevent the following scenario:
273  * pX sleeps on cpuA, leaving its page tables in mmul1
274  * pX wakes up on cpuB, and exits, freeing its page tables
275  * pY on cpuB allocates a freed page table page and overwrites with data
276  * cpuA takes an interrupt, and is now running with bad page tables
277  * In theory this shouldn't hurt because only user address space tables
278  * are affected, and mmuswitch will clear mmul1 before a user process is
279  * dispatched. But empirically it correlates with weird problems, eg
280  * resetting of the core clock at 0x4000001C which confuses local timers.
281  */
282  if(conf.nmach > 1)
283  mmuswitch(nil);
284 
285  if(p->asid > 0)
286  p->asid = -p->asid;
287 }
288 
289 void
290 putmmu(uintptr va, uintptr pa, Page *pg)
291 {
292  uintptr *pte, old;
293  int s;
294 
295  s = splhi();
296  while((pte = mmuwalk(va, 0)) == nil){
297  spllo();
298  assert(up->mmufree == nil);
299  up->mmufree = newpage(0, nil, 0);
300  splhi();
301  }
302  old = *pte;
303  *pte = 0;
304  if((old & PTEVALID) != 0)
305  flushasidvall((uvlong)up->asid<<48 | va>>12);
306  else
307  flushasidva((uvlong)up->asid<<48 | va>>12);
308  *pte = pa | PTEPAGE | PTEUSER | PTENG | PTEAF | PTESH(SHARE_INNER);
309  if(pg->txtflush & (1UL<<m->machno)){
310  /* pio() sets PG_TXTFLUSH whenever a text pg has been written */
311  cachedwbinvse((void*)KADDR(pg->pa), BY2PG);
312  cacheiinvse((void*)va, BY2PG);
313  pg->txtflush &= ~(1UL<<m->machno);
314  }
315  splx(s);
316 }
317 
318 static void
319 mmufree(Proc *p)
320 {
321  freeasid(p);
322 
323  if(p->mmul1 == nil){
324  assert(p->mmul2 == nil);
325  return;
326  }
327  p->mmul1tail->next = p->mmufree;
328  p->mmufree = p->mmul1;
329  p->mmul1 = p->mmul1tail = nil;
330 
331  if(PTLEVELS > 2){
332  p->mmul2tail->next = p->mmufree;
333  p->mmufree = p->mmul2;
334  p->mmul2 = p->mmul2tail = nil;
335  }
336 }
337 
338 void
339 mmuswitch(Proc *p)
340 {
341  uintptr va;
342  Page *t;
343 
344  for(va = UZERO; va < USTKTOP; va += PGLSZ(PTLEVELS-1))
345  m->mmul1[PTL1X(va, PTLEVELS-1)] = 0;
346 
347  if(p == nil){
348  setttbr(PADDR(&m->mmul1[L1TABLEX(0, PTLEVELS-1)]));
349  return;
350  }
351 
352  if(p->newtlb){
353  mmufree(p);
354  p->newtlb = 0;
355  }
356 
357  if(PTLEVELS == 2){
358  for(t = p->mmul1; t != nil; t = t->next){
359  va = t->va;
360  m->mmul1[PTL1X(va, 1)] = t->pa | PTEVALID | PTETABLE;
361  }
362  } else {
363  for(t = p->mmul2; t != nil; t = t->next){
364  va = t->va;
365  m->mmul1[PTL1X(va, 2)] = t->pa | PTEVALID | PTETABLE;
366  if(PTLEVELS > 3)
367  m->mmul1[PTL1X(va, 3)] = PADDR(&m->mmul1[L1TABLEX(va, 2)]) |
368  PTEVALID | PTETABLE;
369  }
370  }
371 
372  if(allocasid(p))
373  flushasid((uvlong)p->asid<<48);
374 
375  setttbr((uvlong)p->asid<<48 | PADDR(&m->mmul1[L1TABLEX(0, PTLEVELS-1)]));
376 }
377 
378 void
379 mmurelease(Proc *p)
380 {
381  Page *t;
382 
383  mmuswitch(nil);
384  mmufree(p);
385 
386  if((t = p->mmufree) != nil){
387  do {
388  p->mmufree = t->next;
389  if(--t->ref != 0)
390  panic("mmurelease: bad page ref");
391  pagechainhead(t);
392  } while((t = p->mmufree) != nil);
393  pagechaindone();
394  }
395 }
396 
397 void
398 flushmmu(void)
399 {
400  int x;
401 
402  x = splhi();
403  up->newtlb = 1;
404  mmuswitch(up);
405  splx(x);
406 }
407 
408 void
409 checkmmu(uintptr, uintptr)
410 {
411 }