changelog shortlog tags branches files raw gz bz2 help

Mercurial > hg > plan9front / changeset: bcm64: add pci express driver for raspberry pi 4

changeset 7315: 6e8b95bccec4
parent 7314: 5afd2605d0f3
child 7316: 234f51ace8cd
author: cinap_lenrek@felloff.net
date: Thu, 25 Jul 2019 09:04:50 +0200
files: sys/src/9/bcm64/dat.h sys/src/9/bcm64/fns.h sys/src/9/bcm64/mmu.c sys/src/9/bcm64/pci.c
description: bcm64: add pci express driver for raspberry pi 4
     1.1--- a/sys/src/9/bcm64/dat.h
     1.2+++ b/sys/src/9/bcm64/dat.h
     1.3@@ -24,6 +24,7 @@ typedef struct MMMU	MMMU;
     1.4 typedef struct Mach	Mach;
     1.5 typedef struct Page	Page;
     1.6 typedef struct PhysUart	PhysUart;
     1.7+typedef struct Pcidev	Pcidev;
     1.8 typedef struct PMMU	PMMU;
     1.9 typedef struct Proc	Proc;
    1.10 typedef u64int		PTE;
    1.11@@ -33,6 +34,7 @@ typedef struct Ureg	Ureg;
    1.12 typedef uvlong		Tval;
    1.13 typedef void		KMap;
    1.14 
    1.15+#pragma incomplete Pcidev
    1.16 #pragma incomplete Ureg
    1.17 
    1.18 #define MAXSYSARG	5	/* for mount(fd, mpt, flag, arg, srv) */
     2.1--- a/sys/src/9/bcm64/fns.h
     2.2+++ b/sys/src/9/bcm64/fns.h
     2.3@@ -68,6 +68,8 @@ extern void kmapinval(void);
     2.4 extern KMap *kmap(Page*);
     2.5 extern void kunmap(KMap*);
     2.6 extern uintptr mmukmap(uintptr, uintptr, usize);
     2.7+extern void* vmap(uintptr, int);
     2.8+extern void vunmap(void*, int);
     2.9 
    2.10 extern void mmu0init(uintptr*);
    2.11 extern void mmu0clear(uintptr*);
    2.12@@ -173,3 +175,29 @@ extern void writeconf(void);
    2.13 extern void screeninit(void);
    2.14 
    2.15 extern int isaconfig(char*, int, ISAConf*);
    2.16+
    2.17+/* pci */
    2.18+typedef struct Pcidev Pcidev;
    2.19+extern int pcicfgr32(Pcidev* pcidev, int rno);
    2.20+extern void pcicfgw32(Pcidev* pcidev, int rno, int data);
    2.21+extern int pcicfgr16(Pcidev* pcidev, int rno);
    2.22+extern void pcicfgw16(Pcidev* pcidev, int rno, int data);
    2.23+extern int pcicfgr8(Pcidev* pcidev, int rno);
    2.24+extern void pcicfgw8(Pcidev* pcidev, int rno, int data);
    2.25+extern Pcidev* pcimatch(Pcidev* prev, int vid, int did);
    2.26+extern Pcidev* pcimatchtbdf(int tbdf);
    2.27+extern void pcisetioe(Pcidev* p);
    2.28+extern void pciclrioe(Pcidev* p);
    2.29+extern void pcisetbme(Pcidev* p);
    2.30+extern void pciclrbme(Pcidev* p);
    2.31+extern void pcisetmwi(Pcidev* p);
    2.32+extern void pciclrmwi(Pcidev* p);
    2.33+extern int pcicap(Pcidev *p, int cap);
    2.34+extern int pcinextcap(Pcidev *pci, int offset);
    2.35+extern int pcihtcap(Pcidev *p, int cap);
    2.36+extern int pcigetpms(Pcidev* p);
    2.37+extern int pcisetpms(Pcidev* p, int state);
    2.38+extern void pcienable(Pcidev *p);
    2.39+extern void pcidisable(Pcidev *p);
    2.40+extern void pciintrenable(int tbdf, void (*f)(Ureg*, void*), void *a);
    2.41+extern void pciintrdisable(int tbdf, void (*f)(Ureg*, void*), void *a);
     3.1--- a/sys/src/9/bcm64/mmu.c
     3.2+++ b/sys/src/9/bcm64/mmu.c
     3.3@@ -26,46 +26,56 @@ mmu0init(uintptr *l1)
     3.4 		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
     3.5 		l1[PTL1X(pa, 1)] = pa | PTEVALID | PTEBLOCK | attr;
     3.6 	}
     3.7-	pe = (uintptr)-KZERO;	/* populate top levels for mmukmap() */
     3.8 	if(PTLEVELS > 2)
     3.9-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2)){
    3.10-		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
    3.11+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
    3.12 		l1[PTL1X(pa, 2)] = (uintptr)&l1[L1TABLEX(pa, 1)] | PTEVALID | PTETABLE;
    3.13-	}
    3.14 	if(PTLEVELS > 3)
    3.15-	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3)){
    3.16-		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
    3.17+	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
    3.18 		l1[PTL1X(pa, 3)] = (uintptr)&l1[L1TABLEX(pa, 2)] | PTEVALID | PTETABLE;
    3.19-	}
    3.20 
    3.21 	/* VIRTIO */
    3.22 	attr = PTEWRITE | PTEAF | PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
    3.23-	pe = soc.physio + IOSIZE;
    3.24-	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
    3.25-		if(pe - pa < PGLSZ(1)){
    3.26+	pe = soc.physio + soc.iosize;
    3.27+	for(pa = soc.physio, va = soc.virtio; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
    3.28+		if(((pa|va) & PGLSZ(1)-1) != 0){
    3.29 			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
    3.30-			for(; pa < pe; pa += PGLSZ(0), va += PGLSZ(0))
    3.31+			for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
    3.32+				assert(l1[PTLX(va, 0)] == 0);
    3.33 				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
    3.34+			}
    3.35 			break;
    3.36 		}
    3.37 		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
    3.38 	}
    3.39-	if(PTLEVELS > 2)
    3.40-	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
    3.41-		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
    3.42-	if(PTLEVELS > 3)
    3.43-	for(pa = soc.physio, va = VIRTIO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
    3.44-		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
    3.45 
    3.46 	/* ARMLOCAL */
    3.47+	attr = PTEWRITE | PTEAF | PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
    3.48 	pe = soc.armlocal + MB;
    3.49-	for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
    3.50+	for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(1), va += PGLSZ(1)){
    3.51+		if(((pa|va) & PGLSZ(1)-1) != 0){
    3.52+			l1[PTL1X(va, 1)] = (uintptr)l1 | PTEVALID | PTETABLE;
    3.53+			for(; pa < pe && ((va|pa) & PGLSZ(1)-1) != 0; pa += PGLSZ(0), va += PGLSZ(0)){
    3.54+				assert(l1[PTLX(va, 0)] == 0);
    3.55+				l1[PTLX(va, 0)] = pa | PTEVALID | PTEPAGE | attr;
    3.56+			}
    3.57+			break;
    3.58+		}
    3.59 		l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
    3.60+	}
    3.61+
    3.62+	/* VIRTPCI */
    3.63+	if(soc.pciwin){
    3.64+		attr = PTEWRITE | PTEAF | PTEKERNEL | PTESH(SHARE_OUTER) | PTEDEVICE;
    3.65+		pe = soc.pciwin + 512*MB;
    3.66+		for(pa = soc.pciwin, va = VIRTPCI; pa < pe; pa += PGLSZ(1), va += PGLSZ(1))
    3.67+			l1[PTL1X(va, 1)] = pa | PTEVALID | PTEBLOCK | attr;
    3.68+	}
    3.69+
    3.70 	if(PTLEVELS > 2)
    3.71-	for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(2), va += PGLSZ(2))
    3.72+	for(va = KSEG0; va != 0; va += PGLSZ(2))
    3.73 		l1[PTL1X(va, 2)] = (uintptr)&l1[L1TABLEX(va, 1)] | PTEVALID | PTETABLE;
    3.74 	if(PTLEVELS > 3)
    3.75-	for(pa = soc.armlocal, va = ARMLOCAL; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
    3.76+	for(va = KSEG0; va != 0; va += PGLSZ(3))
    3.77 		l1[PTL1X(va, 3)] = (uintptr)&l1[L1TABLEX(va, 2)] | PTEVALID | PTETABLE;
    3.78 }
    3.79 
    3.80@@ -107,6 +117,7 @@ mmuidmap(uintptr *l1)
    3.81 	for(pa = PHYSDRAM, va = KZERO; pa < pe; pa += PGLSZ(3), va += PGLSZ(3))
    3.82 		l1[PTL1X(pa, 3)] = l1[PTL1X(va, 3)];
    3.83 	setttbr(PADDR(&l1[L1TABLEX(0, PTLEVELS-1)]));
    3.84+	flushtlb();
    3.85 }
    3.86 
    3.87 void
    3.88@@ -184,6 +195,23 @@ mmukmap(uintptr va, uintptr pa, usize si
    3.89 	return a;
    3.90 }
    3.91 
    3.92+void*
    3.93+vmap(uintptr pa, int)
    3.94+{
    3.95+	if(soc.pciwin && pa >= soc.pciwin)
    3.96+		return (void*)(VIRTPCI + (pa - soc.pciwin));
    3.97+	if(soc.armlocal && pa >= soc.armlocal)
    3.98+		return (void*)(ARMLOCAL + (pa - soc.armlocal));
    3.99+	if(soc.physio && pa >= soc.physio)
   3.100+		return (void*)(soc.virtio + (pa - soc.physio));
   3.101+	return nil;
   3.102+}
   3.103+
   3.104+void
   3.105+vunmap(void *, int)
   3.106+{
   3.107+}
   3.108+
   3.109 static uintptr*
   3.110 mmuwalk(uintptr va, int level)
   3.111 {
     4.1new file mode 100644
     4.2--- /dev/null
     4.3+++ b/sys/src/9/bcm64/pci.c
     4.4@@ -0,0 +1,1129 @@
     4.5+#include "u.h"
     4.6+#include "../port/lib.h"
     4.7+#include "mem.h"
     4.8+#include "dat.h"
     4.9+#include "fns.h"
    4.10+#include "io.h"
    4.11+
    4.12+/* bcmstb PCIe controller registers */
    4.13+enum{
    4.14+	RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1	= 0x0188/4,
    4.15+	RC_CFG_PRIV1_ID_VAL3			= 0x043c/4,
    4.16+	RC_DL_MDIO_ADDR				= 0x1100/4,
    4.17+	RC_DL_MDIO_WR_DATA			= 0x1104/4,
    4.18+	RC_DL_MDIO_RD_DATA			= 0x1108/4,
    4.19+	MISC_MISC_CTRL				= 0x4008/4,
    4.20+	MISC_CPU_2_PCIE_MEM_WIN0_LO		= 0x400c/4,
    4.21+	MISC_CPU_2_PCIE_MEM_WIN0_HI		= 0x4010/4,
    4.22+	MISC_RC_BAR1_CONFIG_LO			= 0x402c/4,
    4.23+	MISC_RC_BAR2_CONFIG_LO			= 0x4034/4,
    4.24+	MISC_RC_BAR2_CONFIG_HI			= 0x4038/4,
    4.25+	MISC_RC_BAR3_CONFIG_LO			= 0x403c/4,
    4.26+	MISC_MSI_BAR_CONFIG_LO			= 0x4044/4,
    4.27+	MISC_MSI_BAR_CONFIG_HI			= 0x4048/4,
    4.28+	MISC_MSI_DATA_CONFIG			= 0x404c/4,
    4.29+	MISC_EOI_CTRL				= 0x4060/4,
    4.30+	MISC_PCIE_CTRL				= 0x4064/4,
    4.31+	MISC_PCIE_STATUS			= 0x4068/4,
    4.32+	MISC_REVISION				= 0x406c/4,
    4.33+	MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT	= 0x4070/4,
    4.34+	MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI	= 0x4080/4,
    4.35+	MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI	= 0x4084/4,
    4.36+	MISC_HARD_PCIE_HARD_DEBUG		= 0x4204/4,
    4.37+
    4.38+	INTR2_CPU_BASE				= 0x4300/4,
    4.39+	MSI_INTR2_BASE				= 0x4500/4,
    4.40+		INTR_STATUS = 0,
    4.41+		INTR_SET,
    4.42+		INTR_CLR,
    4.43+		INTR_MASK_STATUS,
    4.44+		INTR_MASK_SET,
    4.45+		INTR_MASK_CLR,
    4.46+
    4.47+	EXT_CFG_INDEX				= 0x9000/4,
    4.48+	RGR1_SW_INIT_1				= 0x9210/4,
    4.49+	EXT_CFG_DATA				= 0x8000/4,
    4.50+
    4.51+};
    4.52+
    4.53+#define MSI_TARGET_ADDR		0xFFFFFFFFCULL
    4.54+
    4.55+static u32int *regs = (u32int*)(VIRTIO1 + 0x500000);
    4.56+
    4.57+static Lock pcicfglock;
    4.58+static int pcimaxbno = 0;
    4.59+static int pcimaxdno = 0;
    4.60+static Pcidev* pciroot;
    4.61+static Pcidev* pcilist;
    4.62+static Pcidev* pcitail;
    4.63+
    4.64+typedef struct Pcisiz Pcisiz;
    4.65+struct Pcisiz
    4.66+{
    4.67+	Pcidev*	dev;
    4.68+	int	siz;
    4.69+	int	bar;
    4.70+};
    4.71+
    4.72+enum
    4.73+{
    4.74+	MaxFNO		= 7,
    4.75+	MaxUBN		= 255,
    4.76+};
    4.77+
    4.78+static char* bustypes[] = {
    4.79+	"CBUSI",
    4.80+	"CBUSII",
    4.81+	"EISA",
    4.82+	"FUTURE",
    4.83+	"INTERN",
    4.84+	"ISA",
    4.85+	"MBI",
    4.86+	"MBII",
    4.87+	"MCA",
    4.88+	"MPI",
    4.89+	"MPSA",
    4.90+	"NUBUS",
    4.91+	"PCI",
    4.92+	"PCMCIA",
    4.93+	"TC",
    4.94+	"VL",
    4.95+	"VME",
    4.96+	"XPRESS",
    4.97+};
    4.98+
    4.99+static int
   4.100+tbdffmt(Fmt* fmt)
   4.101+{
   4.102+	char *p;
   4.103+	int l, r;
   4.104+	uint type, tbdf;
   4.105+
   4.106+	if((p = malloc(READSTR)) == nil)
   4.107+		return fmtstrcpy(fmt, "(tbdfconv)");
   4.108+
   4.109+	switch(fmt->r){
   4.110+	case 'T':
   4.111+		tbdf = va_arg(fmt->args, int);
   4.112+		if(tbdf == BUSUNKNOWN)
   4.113+			snprint(p, READSTR, "unknown");
   4.114+		else{
   4.115+			type = BUSTYPE(tbdf);
   4.116+			if(type < nelem(bustypes))
   4.117+				l = snprint(p, READSTR, bustypes[type]);
   4.118+			else
   4.119+				l = snprint(p, READSTR, "%d", type);
   4.120+			snprint(p+l, READSTR-l, ".%d.%d.%d",
   4.121+				BUSBNO(tbdf), BUSDNO(tbdf), BUSFNO(tbdf));
   4.122+		}
   4.123+		break;
   4.124+
   4.125+	default:
   4.126+		snprint(p, READSTR, "(tbdfconv)");
   4.127+		break;
   4.128+	}
   4.129+	r = fmtstrcpy(fmt, p);
   4.130+	free(p);
   4.131+
   4.132+	return r;
   4.133+}
   4.134+
   4.135+static void pcicfginit(void);
   4.136+
   4.137+static void*
   4.138+cfgaddr(int tbdf, int rno)
   4.139+{
   4.140+	if(BUSBNO(tbdf) == 0 && BUSDNO(tbdf) == 0)
   4.141+		return (uchar*)regs + rno;
   4.142+	regs[EXT_CFG_INDEX] = BUSBNO(tbdf) << 20 | BUSDNO(tbdf) << 15 | BUSFNO(tbdf) << 12;
   4.143+	coherence();
   4.144+	return ((uchar*)&regs[EXT_CFG_DATA]) + rno;
   4.145+}
   4.146+
   4.147+static int
   4.148+pcicfgrw32(int tbdf, int rno, int data, int read)
   4.149+{
   4.150+	int x = -1;
   4.151+	u32int *p;
   4.152+
   4.153+	ilock(&pcicfglock);
   4.154+	if((p = cfgaddr(tbdf, rno & ~3)) != nil){
   4.155+		if(read)
   4.156+			x = *p;
   4.157+		else
   4.158+			*p = data;
   4.159+	}
   4.160+	iunlock(&pcicfglock);
   4.161+	return x;
   4.162+}
   4.163+static int
   4.164+pcicfgrw16(int tbdf, int rno, int data, int read)
   4.165+{
   4.166+	int x = -1;
   4.167+	u16int *p;
   4.168+
   4.169+	ilock(&pcicfglock);
   4.170+	if((p = cfgaddr(tbdf, rno & ~1)) != nil){
   4.171+		if(read)
   4.172+			x = *p;
   4.173+		else
   4.174+			*p = data;
   4.175+	}
   4.176+	iunlock(&pcicfglock);
   4.177+	return x;
   4.178+}
   4.179+static int
   4.180+pcicfgrw8(int tbdf, int rno, int data, int read)
   4.181+{
   4.182+	int x = -1;
   4.183+	u8int *p;
   4.184+
   4.185+	ilock(&pcicfglock);
   4.186+	if((p = cfgaddr(tbdf, rno)) != nil){
   4.187+		if(read)
   4.188+			x = *p;
   4.189+		else
   4.190+			*p = data;
   4.191+	}
   4.192+	iunlock(&pcicfglock);
   4.193+	return x;
   4.194+}
   4.195+
   4.196+int
   4.197+pcicfgr32(Pcidev* pcidev, int rno)
   4.198+{
   4.199+	return pcicfgrw32(pcidev->tbdf, rno, 0, 1);
   4.200+}
   4.201+void
   4.202+pcicfgw32(Pcidev* pcidev, int rno, int data)
   4.203+{
   4.204+	pcicfgrw32(pcidev->tbdf, rno, data, 0);
   4.205+}
   4.206+int
   4.207+pcicfgr16(Pcidev* pcidev, int rno)
   4.208+{
   4.209+	return pcicfgrw16(pcidev->tbdf, rno, 0, 1);
   4.210+}
   4.211+void
   4.212+pcicfgw16(Pcidev* pcidev, int rno, int data)
   4.213+{
   4.214+	pcicfgrw16(pcidev->tbdf, rno, data, 0);
   4.215+}
   4.216+int
   4.217+pcicfgr8(Pcidev* pcidev, int rno)
   4.218+{
   4.219+	return pcicfgrw8(pcidev->tbdf, rno, 0, 1);
   4.220+}
   4.221+void
   4.222+pcicfgw8(Pcidev* pcidev, int rno, int data)
   4.223+{
   4.224+	pcicfgrw8(pcidev->tbdf, rno, data, 0);
   4.225+}
   4.226+
   4.227+Pcidev*
   4.228+pcimatch(Pcidev* prev, int vid, int did)
   4.229+{
   4.230+	if(prev == nil)
   4.231+		prev = pcilist;
   4.232+	else
   4.233+		prev = prev->list;
   4.234+
   4.235+	while(prev != nil){
   4.236+		if((vid == 0 || prev->vid == vid)
   4.237+		&& (did == 0 || prev->did == did))
   4.238+			break;
   4.239+		prev = prev->list;
   4.240+	}
   4.241+	return prev;
   4.242+}
   4.243+
   4.244+Pcidev*
   4.245+pcimatchtbdf(int tbdf)
   4.246+{
   4.247+	Pcidev *pcidev;
   4.248+
   4.249+	for(pcidev = pcilist; pcidev != nil; pcidev = pcidev->list) {
   4.250+		if(pcidev->tbdf == tbdf)
   4.251+			break;
   4.252+	}
   4.253+	return pcidev;
   4.254+}
   4.255+
   4.256+static u32int
   4.257+pcibarsize(Pcidev *p, int rno)
   4.258+{
   4.259+	u32int v, size;
   4.260+
   4.261+	v = pcicfgrw32(p->tbdf, rno, 0, 1);
   4.262+	pcicfgrw32(p->tbdf, rno, 0xFFFFFFF0, 0);
   4.263+	size = pcicfgrw32(p->tbdf, rno, 0, 1);
   4.264+	if(v & 1)
   4.265+		size |= 0xFFFF0000;
   4.266+	pcicfgrw32(p->tbdf, rno, v, 0);
   4.267+
   4.268+	return -(size & ~0x0F);
   4.269+}
   4.270+
   4.271+static int
   4.272+pcisizcmp(void *a, void *b)
   4.273+{
   4.274+	Pcisiz *aa, *bb;
   4.275+
   4.276+	aa = a;
   4.277+	bb = b;
   4.278+	return aa->siz - bb->siz;
   4.279+}
   4.280+
   4.281+static ulong
   4.282+pcimask(ulong v)
   4.283+{
   4.284+	ulong m;
   4.285+
   4.286+	m = BI2BY*sizeof(v);
   4.287+	for(m = 1<<(m-1); m != 0; m >>= 1) {
   4.288+		if(m & v)
   4.289+			break;
   4.290+	}
   4.291+
   4.292+	m--;
   4.293+	if((v & m) == 0)
   4.294+		return v;
   4.295+
   4.296+	v |= m;
   4.297+	return v+1;
   4.298+}
   4.299+
   4.300+static void
   4.301+pcibusmap(Pcidev *root, uintptr *pmema, uintptr *pioa, int wrreg)
   4.302+{
   4.303+	Pcidev *p;
   4.304+	int ntb, i, size, rno, hole;
   4.305+	uintptr v, mema, ioa, sioa, smema, base, limit;
   4.306+	Pcisiz *table, *tptr, *mtb, *itb;
   4.307+
   4.308+	ioa = *pioa;
   4.309+	mema = *pmema;
   4.310+
   4.311+	ntb = 0;
   4.312+	for(p = root; p != nil; p = p->link)
   4.313+		ntb++;
   4.314+
   4.315+	ntb *= (PciCIS-PciBAR0)/4;
   4.316+	table = malloc(2*ntb*sizeof(Pcisiz));
   4.317+	if(table == nil)
   4.318+		panic("pcibusmap: can't allocate memory");
   4.319+	itb = table;
   4.320+	mtb = table+ntb;
   4.321+
   4.322+	/*
   4.323+	 * Build a table of sizes
   4.324+	 */
   4.325+	for(p = root; p != nil; p = p->link) {
   4.326+		if(p->ccrb == 0x06) {
   4.327+			if(p->ccru != 0x04 || p->bridge == nil)
   4.328+				continue;
   4.329+
   4.330+			sioa = ioa;
   4.331+			smema = mema;
   4.332+			pcibusmap(p->bridge, &smema, &sioa, 0);
   4.333+
   4.334+			hole = pcimask(smema-mema);
   4.335+			if(hole < (1<<20))
   4.336+				hole = 1<<20;
   4.337+			p->mema.size = hole;
   4.338+
   4.339+			hole = pcimask(sioa-ioa);
   4.340+			if(hole < (1<<12))
   4.341+				hole = 1<<12;
   4.342+
   4.343+			p->ioa.size = hole;
   4.344+
   4.345+			itb->dev = p;
   4.346+			itb->bar = -1;
   4.347+			itb->siz = p->ioa.size;
   4.348+			itb++;
   4.349+
   4.350+			mtb->dev = p;
   4.351+			mtb->bar = -1;
   4.352+			mtb->siz = p->mema.size;
   4.353+			mtb++;
   4.354+			continue;
   4.355+		}
   4.356+
   4.357+		for(i = 0; i <= 5; i++) {
   4.358+			rno = PciBAR0 + i*4;
   4.359+			v = pcicfgrw32(p->tbdf, rno, 0, 1);
   4.360+			size = pcibarsize(p, rno);
   4.361+			if(size == 0)
   4.362+				continue;
   4.363+
   4.364+			p->mem[i].size = size;
   4.365+			if(v & 1) {
   4.366+				itb->dev = p;
   4.367+				itb->bar = i;
   4.368+				itb->siz = size;
   4.369+				itb++;
   4.370+			}
   4.371+			else {
   4.372+				mtb->dev = p;
   4.373+				mtb->bar = i;
   4.374+				mtb->siz = size;
   4.375+				mtb++;
   4.376+
   4.377+				if((v & 7) == 4)
   4.378+					i++;
   4.379+			}
   4.380+		}
   4.381+	}
   4.382+
   4.383+	/*
   4.384+	 * Sort both tables IO smallest first, Memory largest
   4.385+	 */
   4.386+	qsort(table, itb-table, sizeof(Pcisiz), pcisizcmp);
   4.387+	tptr = table+ntb;
   4.388+	qsort(tptr, mtb-tptr, sizeof(Pcisiz), pcisizcmp);
   4.389+
   4.390+	/*
   4.391+	 * Allocate IO address space on this bus
   4.392+	 */
   4.393+	for(tptr = table; tptr < itb; tptr++) {
   4.394+		hole = tptr->siz;
   4.395+		if(tptr->bar == -1)
   4.396+			hole = 1<<12;
   4.397+		ioa = (ioa+hole-1) & ~(hole-1);
   4.398+
   4.399+		p = tptr->dev;
   4.400+		if(tptr->bar == -1)
   4.401+			p->ioa.bar = ioa;
   4.402+		else {
   4.403+			p->pcr |= IOen;
   4.404+			p->mem[tptr->bar].bar = ioa|1;
   4.405+			if(wrreg)
   4.406+				pcicfgrw32(p->tbdf, PciBAR0+(tptr->bar*4), ioa|1, 0);
   4.407+		}
   4.408+
   4.409+		ioa += tptr->siz;
   4.410+	}
   4.411+
   4.412+	/*
   4.413+	 * Allocate Memory address space on this bus
   4.414+	 */
   4.415+	for(tptr = table+ntb; tptr < mtb; tptr++) {
   4.416+		hole = tptr->siz;
   4.417+		if(tptr->bar == -1)
   4.418+			hole = 1<<20;
   4.419+		mema = (mema+hole-1) & ~(hole-1);
   4.420+
   4.421+		p = tptr->dev;
   4.422+		if(tptr->bar == -1)
   4.423+			p->mema.bar = mema;
   4.424+		else {
   4.425+			p->pcr |= MEMen;
   4.426+			p->mem[tptr->bar].bar = mema;
   4.427+			if(wrreg){
   4.428+				rno = PciBAR0+(tptr->bar*4);
   4.429+				if((mema >> 32) != 0){
   4.430+					pcicfgrw32(p->tbdf, rno, mema|4, 0);
   4.431+					pcicfgrw32(p->tbdf, rno+4, mema >> 32, 0);
   4.432+				} else {
   4.433+					pcicfgrw32(p->tbdf, rno, mema, 0);
   4.434+				}
   4.435+			}
   4.436+		}
   4.437+		mema += tptr->siz;
   4.438+	}
   4.439+
   4.440+	*pmema = mema;
   4.441+	*pioa = ioa;
   4.442+	free(table);
   4.443+
   4.444+	if(wrreg == 0)
   4.445+		return;
   4.446+
   4.447+	/*
   4.448+	 * Finally set all the bridge addresses & registers
   4.449+	 */
   4.450+	for(p = root; p != nil; p = p->link) {
   4.451+		if(p->bridge == nil) {
   4.452+			if(p->cls == 0){
   4.453+				p->cls = 64;
   4.454+				pcicfgw8(p, PciCLS, p->cls);
   4.455+			}
   4.456+			pcicfgrw8(p->tbdf, PciLTR, 64, 0);
   4.457+			p->pcr |= MASen;
   4.458+			pcicfgrw16(p->tbdf, PciPCR, p->pcr, 0);
   4.459+			continue;
   4.460+		}
   4.461+
   4.462+		if(p == pciroot){
   4.463+			base = p->mema.bar;
   4.464+			limit = base+p->mema.size-1;
   4.465+			regs[MISC_CPU_2_PCIE_MEM_WIN0_LO] = base;
   4.466+			regs[MISC_CPU_2_PCIE_MEM_WIN0_HI] = base >> 32;
   4.467+			base >>= 20, limit >>= 20;
   4.468+			regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT] = (base & 0xFFF) << 4 | (limit & 0xFFF) << 20;
   4.469+			regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI] = base >> 12;
   4.470+			regs[MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI] = limit >> 12;
   4.471+		}
   4.472+
   4.473+		base = p->ioa.bar;
   4.474+		limit = base+p->ioa.size-1;
   4.475+		v = pcicfgrw32(p->tbdf, PciIBR, 0, 1);
   4.476+		v = (v&0xFFFF0000)|(limit & 0xF000)|((base & 0xF000)>>8);
   4.477+		pcicfgrw32(p->tbdf, PciIBR, v, 0);
   4.478+		v = (limit & 0xFFFF0000)|(base>>16);
   4.479+		pcicfgrw32(p->tbdf, PciIUBR, v, 0);
   4.480+
   4.481+		base = p->mema.bar;
   4.482+		limit = base+p->mema.size-1;
   4.483+		v = (limit & 0xFFF00000)|((base & 0xFFF00000)>>16);
   4.484+		pcicfgrw32(p->tbdf, PciMBR, v, 0);
   4.485+
   4.486+		/*
   4.487+		 * Disable memory prefetch
   4.488+		 */
   4.489+		pcicfgrw32(p->tbdf, PciPMBR, 0x0000FFFF, 0);
   4.490+		pcicfgrw8(p->tbdf, PciLTR, 64, 0);
   4.491+
   4.492+		/*
   4.493+		 * Enable the bridge
   4.494+		 */
   4.495+		p->pcr |= IOen|MEMen|MASen;
   4.496+		pcicfgrw32(p->tbdf, PciPCR, 0xFFFF0000|p->pcr, 0);
   4.497+
   4.498+		sioa = p->ioa.bar;
   4.499+		smema = p->mema.bar;
   4.500+		pcibusmap(p->bridge, &smema, &sioa, 1);
   4.501+	}
   4.502+}
   4.503+
   4.504+static int
   4.505+pcilscan(int bno, Pcidev** list, Pcidev *parent)
   4.506+{
   4.507+	Pcidev *p, *head, *tail;
   4.508+	int dno, fno, i, hdt, l, maxfno, maxubn, rno, sbn, tbdf, ubn;
   4.509+
   4.510+	maxubn = bno;
   4.511+	head = nil;
   4.512+	tail = nil;
   4.513+	for(dno = 0; dno <= pcimaxdno; dno++){
   4.514+		maxfno = 0;
   4.515+		for(fno = 0; fno <= maxfno; fno++){
   4.516+			/*
   4.517+			 * For this possible device, form the
   4.518+			 * bus+device+function triplet needed to address it
   4.519+			 * and try to read the vendor and device ID.
   4.520+			 * If successful, allocate a device struct and
   4.521+			 * start to fill it in with some useful information
   4.522+			 * from the device's configuration space.
   4.523+			 */
   4.524+			tbdf = MKBUS(BusPCI, bno, dno, fno);
   4.525+			l = pcicfgrw32(tbdf, PciVID, 0, 1);
   4.526+			if(l == 0xFFFFFFFF || l == 0)
   4.527+				continue;
   4.528+			p = malloc(sizeof(*p));
   4.529+			if(p == nil)
   4.530+				panic("pcilscan: no memory");
   4.531+			p->tbdf = tbdf;
   4.532+			p->vid = l;
   4.533+			p->did = l>>16;
   4.534+
   4.535+			if(pcilist != nil)
   4.536+				pcitail->list = p;
   4.537+			else
   4.538+				pcilist = p;
   4.539+			pcitail = p;
   4.540+
   4.541+			p->pcr = pcicfgr16(p, PciPCR);
   4.542+			p->rid = pcicfgr8(p, PciRID);
   4.543+			p->ccrp = pcicfgr8(p, PciCCRp);
   4.544+			p->ccru = pcicfgr8(p, PciCCRu);
   4.545+			p->ccrb = pcicfgr8(p, PciCCRb);
   4.546+			p->cls = pcicfgr8(p, PciCLS);
   4.547+			p->ltr = pcicfgr8(p, PciLTR);
   4.548+
   4.549+			p->intl = pcicfgr8(p, PciINTL);
   4.550+
   4.551+			/*
   4.552+			 * If the device is a multi-function device adjust the
   4.553+			 * loop count so all possible functions are checked.
   4.554+			 */
   4.555+			hdt = pcicfgr8(p, PciHDT);
   4.556+			if(hdt & 0x80)
   4.557+				maxfno = MaxFNO;
   4.558+
   4.559+			/*
   4.560+			 * If appropriate, read the base address registers
   4.561+			 * and work out the sizes.
   4.562+			 */
   4.563+			switch(p->ccrb) {
   4.564+			case 0x00:		/* prehistoric */
   4.565+			case 0x01:		/* mass storage controller */
   4.566+			case 0x02:		/* network controller */
   4.567+			case 0x03:		/* display controller */
   4.568+			case 0x04:		/* multimedia device */
   4.569+			case 0x07:		/* simple comm. controllers */
   4.570+			case 0x08:		/* base system peripherals */
   4.571+			case 0x09:		/* input devices */
   4.572+			case 0x0A:		/* docking stations */
   4.573+			case 0x0B:		/* processors */
   4.574+			case 0x0C:		/* serial bus controllers */
   4.575+			case 0x0D:		/* wireless controllers */
   4.576+			case 0x0E:		/* intelligent I/O controllers */
   4.577+			case 0x0F:		/* sattelite communication controllers */
   4.578+			case 0x10:		/* encryption/decryption controllers */
   4.579+			case 0x11:		/* signal processing controllers */
   4.580+				if((hdt & 0x7F) != 0)
   4.581+					break;
   4.582+				rno = PciBAR0;
   4.583+				for(i = 0; i <= 5; i++) {
   4.584+					p->mem[i].bar = pcicfgr32(p, rno);
   4.585+					p->mem[i].size = pcibarsize(p, rno);
   4.586+					if((p->mem[i].bar & 7) == 4 && i < 5){
   4.587+						rno += 4;
   4.588+						p->mem[i].bar |= (uintptr)pcicfgr32(p, rno) << 32;
   4.589+						i++;
   4.590+					}
   4.591+					rno += 4;
   4.592+				}
   4.593+				break;
   4.594+
   4.595+			case 0x05:		/* memory controller */
   4.596+			case 0x06:		/* bridge device */
   4.597+			default:
   4.598+				break;
   4.599+			}
   4.600+
   4.601+			p->parent = parent;
   4.602+			if(head != nil)
   4.603+				tail->link = p;
   4.604+			else
   4.605+				head = p;
   4.606+			tail = p;
   4.607+		}
   4.608+	}
   4.609+
   4.610+	*list = head;
   4.611+	for(p = head; p != nil; p = p->link){
   4.612+		/*
   4.613+		 * Find PCI-PCI bridges and recursively descend the tree.
   4.614+		 */
   4.615+		if(p->ccrb != 0x06 || p->ccru != 0x04)
   4.616+			continue;
   4.617+
   4.618+		/*
   4.619+		 * If the secondary or subordinate bus number is not
   4.620+		 * initialised try to do what the PCI BIOS should have
   4.621+		 * done and fill in the numbers as the tree is descended.
   4.622+		 * On the way down the subordinate bus number is set to
   4.623+		 * the maximum as it's not known how many buses are behind
   4.624+		 * this one; the final value is set on the way back up.
   4.625+		 */
   4.626+		sbn = pcicfgr8(p, PciSBN);
   4.627+		ubn = pcicfgr8(p, PciUBN);
   4.628+
   4.629+		if(sbn == 0 || ubn == 0) {
   4.630+			sbn = maxubn+1;
   4.631+			/*
   4.632+			 * Make sure memory, I/O and master enables are
   4.633+			 * off, set the primary, secondary and subordinate
   4.634+			 * bus numbers and clear the secondary status before
   4.635+			 * attempting to scan the secondary bus.
   4.636+			 *
   4.637+			 * Initialisation of the bridge should be done here.
   4.638+			 */
   4.639+			pcicfgw32(p, PciPCR, 0xFFFF0000);
   4.640+			l = (MaxUBN<<16)|(sbn<<8)|bno;
   4.641+			pcicfgw32(p, PciPBN, l);
   4.642+			pcicfgw16(p, PciSPSR, 0xFFFF);
   4.643+			maxubn = pcilscan(sbn, &p->bridge, p);
   4.644+			l = (maxubn<<16)|(sbn<<8)|bno;
   4.645+
   4.646+			pcicfgw32(p, PciPBN, l);
   4.647+		}
   4.648+		else {
   4.649+			if(ubn > maxubn)
   4.650+				maxubn = ubn;
   4.651+			pcilscan(sbn, &p->bridge, p);
   4.652+		}
   4.653+	}
   4.654+
   4.655+	return maxubn;
   4.656+}
   4.657+
   4.658+static void
   4.659+pcicfginit(void)
   4.660+{
   4.661+	uintptr mema, ioa;
   4.662+
   4.663+	fmtinstall('T', tbdffmt);
   4.664+
   4.665+	pcilscan(0, &pciroot, nil);
   4.666+
   4.667+	/*
   4.668+	 * Work out how big the top bus is
   4.669+	 */
   4.670+	ioa = 0;
   4.671+	mema = 0;
   4.672+	pcibusmap(pciroot, &mema, &ioa, 0);
   4.673+
   4.674+	/*
   4.675+	 * Align the windows and map it
   4.676+	 */
   4.677+	ioa = 0;
   4.678+	mema = soc.pciwin;
   4.679+	pcibusmap(pciroot, &mema, &ioa, 1);
   4.680+}
   4.681+
   4.682+static void
   4.683+pcilhinv(Pcidev* p)
   4.684+{
   4.685+	int i;
   4.686+	Pcidev *t;
   4.687+
   4.688+	if(p == nil) {
   4.689+		p = pciroot;
   4.690+		print("bus dev type vid  did intl memory\n");
   4.691+	}
   4.692+	for(t = p; t != nil; t = t->link) {
   4.693+		print("%d  %2d/%d %.2ux %.2ux %.2ux %.4ux %.4ux %3d  ",
   4.694+			BUSBNO(t->tbdf), BUSDNO(t->tbdf), BUSFNO(t->tbdf),
   4.695+			t->ccrb, t->ccru, t->ccrp, t->vid, t->did, t->intl);
   4.696+
   4.697+		for(i = 0; i < nelem(p->mem); i++) {
   4.698+			if(t->mem[i].size == 0)
   4.699+				continue;
   4.700+			print("%d:%llux %d ", i,
   4.701+				(uvlong)t->mem[i].bar, t->mem[i].size);
   4.702+		}
   4.703+		if(t->bridge)
   4.704+			print("->%d", BUSBNO(t->bridge->tbdf));
   4.705+		print("\n");
   4.706+	}
   4.707+	while(p != nil) {
   4.708+		if(p->bridge != nil)
   4.709+			pcilhinv(p->bridge);
   4.710+		p = p->link;
   4.711+	}
   4.712+}
   4.713+
   4.714+static void
   4.715+pcihinv(Pcidev* p)
   4.716+{
   4.717+	pcilhinv(p);
   4.718+}
   4.719+
   4.720+void
   4.721+pcisetioe(Pcidev* p)
   4.722+{
   4.723+	p->pcr |= IOen;
   4.724+	pcicfgw16(p, PciPCR, p->pcr);
   4.725+}
   4.726+
   4.727+void
   4.728+pciclrioe(Pcidev* p)
   4.729+{
   4.730+	p->pcr &= ~IOen;
   4.731+	pcicfgw16(p, PciPCR, p->pcr);
   4.732+}
   4.733+
   4.734+void
   4.735+pcisetbme(Pcidev* p)
   4.736+{
   4.737+	p->pcr |= MASen;
   4.738+	pcicfgw16(p, PciPCR, p->pcr);
   4.739+}
   4.740+
   4.741+void
   4.742+pciclrbme(Pcidev* p)
   4.743+{
   4.744+	p->pcr &= ~MASen;
   4.745+	pcicfgw16(p, PciPCR, p->pcr);
   4.746+}
   4.747+
   4.748+void
   4.749+pcisetmwi(Pcidev* p)
   4.750+{
   4.751+	p->pcr |= MemWrInv;
   4.752+	pcicfgw16(p, PciPCR, p->pcr);
   4.753+}
   4.754+
   4.755+void
   4.756+pciclrmwi(Pcidev* p)
   4.757+{
   4.758+	p->pcr &= ~MemWrInv;
   4.759+	pcicfgw16(p, PciPCR, p->pcr);
   4.760+}
   4.761+
   4.762+static int
   4.763+enumcaps(Pcidev *p, int (*fmatch)(Pcidev*, int, int, int), int arg)
   4.764+{
   4.765+	int i, r, cap, off;
   4.766+
   4.767+	/* status register bit 4 has capabilities */
   4.768+	if((pcicfgr16(p, PciPSR) & 1<<4) == 0)
   4.769+		return -1;      
   4.770+	switch(pcicfgr8(p, PciHDT) & 0x7F){
   4.771+	default:
   4.772+		return -1;
   4.773+	case 0:                         /* etc */
   4.774+	case 1:                         /* pci to pci bridge */
   4.775+		off = 0x34;
   4.776+		break;
   4.777+	case 2:                         /* cardbus bridge */
   4.778+		off = 0x14;
   4.779+		break;
   4.780+	}
   4.781+	for(i = 48; i--;){
   4.782+		off = pcicfgr8(p, off);
   4.783+		if(off < 0x40 || (off & 3))
   4.784+			break;
   4.785+		off &= ~3;
   4.786+		cap = pcicfgr8(p, off);
   4.787+		if(cap == 0xff)
   4.788+			break;
   4.789+		r = (*fmatch)(p, cap, off, arg);
   4.790+		if(r < 0)
   4.791+			break;
   4.792+		if(r == 0)
   4.793+			return off;
   4.794+		off++;
   4.795+	}
   4.796+	return -1;
   4.797+}
   4.798+
   4.799+static int
   4.800+matchcap(Pcidev *, int cap, int, int arg)
   4.801+{
   4.802+	return cap != arg;
   4.803+}
   4.804+
   4.805+static int
   4.806+matchhtcap(Pcidev *p, int cap, int off, int arg)
   4.807+{
   4.808+	int mask;
   4.809+
   4.810+	if(cap != PciCapHTC)
   4.811+		return 1;
   4.812+	if(arg == 0x00 || arg == 0x20)
   4.813+		mask = 0xE0;
   4.814+	else
   4.815+		mask = 0xF8;
   4.816+	cap = pcicfgr8(p, off+3);
   4.817+	return (cap & mask) != arg;
   4.818+}
   4.819+
   4.820+int
   4.821+pcicap(Pcidev *p, int cap)
   4.822+{
   4.823+	return enumcaps(p, matchcap, cap);
   4.824+}
   4.825+
   4.826+int
   4.827+pcinextcap(Pcidev *pci, int offset)
   4.828+{
   4.829+	if(offset == 0) {
   4.830+		if((pcicfgr16(pci, PciPSR) & (1<<4)) == 0)
   4.831+			return 0; /* no capabilities */
   4.832+		offset = PciCAP-1;
   4.833+	}
   4.834+	return pcicfgr8(pci, offset+1) & ~3;
   4.835+}
   4.836+
   4.837+int
   4.838+pcihtcap(Pcidev *p, int cap)
   4.839+{
   4.840+	return enumcaps(p, matchhtcap, cap);
   4.841+}
   4.842+
   4.843+static int
   4.844+pcigetpmrb(Pcidev* p)
   4.845+{
   4.846+        if(p->pmrb != 0)
   4.847+                return p->pmrb;
   4.848+        return p->pmrb = pcicap(p, PciCapPMG);
   4.849+}
   4.850+
   4.851+int
   4.852+pcigetpms(Pcidev* p)
   4.853+{
   4.854+	int pmcsr, ptr;
   4.855+
   4.856+	if((ptr = pcigetpmrb(p)) == -1)
   4.857+		return -1;
   4.858+
   4.859+	/*
   4.860+	 * Power Management Register Block:
   4.861+	 *  offset 0:	Capability ID
   4.862+	 *	   1:	next item pointer
   4.863+	 *	   2:	capabilities
   4.864+	 *	   4:	control/status
   4.865+	 *	   6:	bridge support extensions
   4.866+	 *	   7:	data
   4.867+	 */
   4.868+	pmcsr = pcicfgr16(p, ptr+4);
   4.869+
   4.870+	return pmcsr & 0x0003;
   4.871+}
   4.872+
   4.873+int
   4.874+pcisetpms(Pcidev* p, int state)
   4.875+{
   4.876+	int ostate, pmc, pmcsr, ptr;
   4.877+
   4.878+	if((ptr = pcigetpmrb(p)) == -1)
   4.879+		return -1;
   4.880+
   4.881+	pmc = pcicfgr16(p, ptr+2);
   4.882+	pmcsr = pcicfgr16(p, ptr+4);
   4.883+	ostate = pmcsr & 0x0003;
   4.884+	pmcsr &= ~0x0003;
   4.885+
   4.886+	switch(state){
   4.887+	default:
   4.888+		return -1;
   4.889+	case 0:
   4.890+		break;
   4.891+	case 1:
   4.892+		if(!(pmc & 0x0200))
   4.893+			return -1;
   4.894+		break;
   4.895+	case 2:
   4.896+		if(!(pmc & 0x0400))
   4.897+			return -1;
   4.898+		break;
   4.899+	case 3:
   4.900+		break;
   4.901+	}
   4.902+	pmcsr |= state;
   4.903+	pcicfgw16(p, ptr+4, pmcsr);
   4.904+
   4.905+	return ostate;
   4.906+}
   4.907+
   4.908+void
   4.909+pcienable(Pcidev *p)
   4.910+{
   4.911+	uint pcr;
   4.912+	int i;
   4.913+
   4.914+	if(p == nil)
   4.915+		return;
   4.916+
   4.917+	pcienable(p->parent);
   4.918+
   4.919+	switch(pcisetpms(p, 0)){
   4.920+	case 1:
   4.921+		print("pcienable %T: wakeup from D1\n", p->tbdf);
   4.922+		break;
   4.923+	case 2:
   4.924+		print("pcienable %T: wakeup from D2\n", p->tbdf);
   4.925+		if(p->bridge != nil)
   4.926+			delay(100);	/* B2: minimum delay 50ms */
   4.927+		else
   4.928+			delay(1);	/* D2: minimum delay 200┬Ás */
   4.929+		break;
   4.930+	case 3:
   4.931+		print("pcienable %T: wakeup from D3\n", p->tbdf);
   4.932+		delay(100);		/* D3: minimum delay 50ms */
   4.933+
   4.934+		/* restore registers */
   4.935+		for(i = 0; i < 6; i++)
   4.936+			pcicfgw32(p, PciBAR0+i*4, p->mem[i].bar);
   4.937+		pcicfgw8(p, PciINTL, p->intl);
   4.938+		pcicfgw8(p, PciLTR, p->ltr);
   4.939+		pcicfgw8(p, PciCLS, p->cls);
   4.940+		pcicfgw16(p, PciPCR, p->pcr);
   4.941+		break;
   4.942+	}
   4.943+
   4.944+	if(p->bridge != nil)
   4.945+		pcr = IOen|MEMen|MASen;
   4.946+	else {
   4.947+		pcr = 0;
   4.948+		for(i = 0; i < 6; i++){
   4.949+			if(p->mem[i].size == 0)
   4.950+				continue;
   4.951+			if(p->mem[i].bar & 1)
   4.952+				pcr |= IOen;
   4.953+			else
   4.954+				pcr |= MEMen;
   4.955+		}
   4.956+	}
   4.957+
   4.958+	if((p->pcr & pcr) != pcr){
   4.959+		print("pcienable %T: pcr %ux->%ux\n", p->tbdf, p->pcr, p->pcr|pcr);
   4.960+		p->pcr |= pcr;
   4.961+		pcicfgrw32(p->tbdf, PciPCR, 0xFFFF0000|p->pcr, 0);
   4.962+	}
   4.963+}
   4.964+
   4.965+void
   4.966+pcidisable(Pcidev *p)
   4.967+{
   4.968+	if(p == nil)
   4.969+		return;
   4.970+	pciclrbme(p);
   4.971+}
   4.972+
   4.973+enum {
   4.974+	MSICtrl = 0x02, /* message control register (16 bit) */
   4.975+	MSIAddr = 0x04, /* message address register (64 bit) */
   4.976+	MSIData32 = 0x08, /* message data register for 32 bit MSI (16 bit) */
   4.977+	MSIData64 = 0x0C, /* message data register for 64 bit MSI (16 bit) */
   4.978+};
   4.979+
   4.980+typedef struct Pciisr Pciisr;
   4.981+struct Pciisr {
   4.982+	void	(*f)(Ureg*, void*);
   4.983+	void	*a;
   4.984+	Pcidev	*p;
   4.985+};
   4.986+
   4.987+static Pciisr pciisr[32];
   4.988+static Lock pciisrlk;
   4.989+
   4.990+void
   4.991+pciintrenable(int tbdf, void (*f)(Ureg*, void*), void *a)
   4.992+{
   4.993+	int cap, ok64;
   4.994+	u32int dat;
   4.995+	u64int adr;
   4.996+	Pcidev *p;
   4.997+	Pciisr *isr;
   4.998+
   4.999+	if((p = pcimatchtbdf(tbdf)) == nil){
  4.1000+		print("pciintrenable: %T: unknown device\n", tbdf);
  4.1001+		return;
  4.1002+	}
  4.1003+	if((cap = pcicap(p, PciCapMSI)) < 0){
  4.1004+		print("pciintrenable: %T: no MSI cap\n", tbdf);
  4.1005+		return;
  4.1006+	}
  4.1007+
  4.1008+	lock(&pciisrlk);
  4.1009+	for(isr = pciisr; isr < &pciisr[nelem(pciisr)]; isr++){
  4.1010+		if(isr->p == p){
  4.1011+			isr->p = nil;
  4.1012+			regs[MSI_INTR2_BASE + INTR_MASK_SET] = 1 << (isr-pciisr);
  4.1013+			break;
  4.1014+		}
  4.1015+	}
  4.1016+	for(isr = pciisr; isr < &pciisr[nelem(pciisr)]; isr++){
  4.1017+		if(isr->p == nil){
  4.1018+			isr->p = p;
  4.1019+			isr->a = a;
  4.1020+			isr->f = f;
  4.1021+			regs[MSI_INTR2_BASE + INTR_CLR] = 1 << (isr-pciisr);
  4.1022+			regs[MSI_INTR2_BASE + INTR_MASK_CLR] = 1 << (isr-pciisr);
  4.1023+			break;
  4.1024+		}
  4.1025+	}
  4.1026+	unlock(&pciisrlk);
  4.1027+
  4.1028+	if(isr >= &pciisr[nelem(pciisr)]){
  4.1029+		print("pciintrenable: %T: out of isr slots\n", tbdf);
  4.1030+		return;
  4.1031+	}
  4.1032+
  4.1033+	adr = MSI_TARGET_ADDR;
  4.1034+	ok64 = (pcicfgr16(p, cap + MSICtrl) & (1<<7)) != 0;
  4.1035+	pcicfgw32(p, cap + MSIAddr, adr);
  4.1036+	if(ok64) pcicfgw32(p, cap + MSIAddr + 4, adr>>32);
  4.1037+	dat = regs[MISC_MSI_DATA_CONFIG];
  4.1038+	dat = ((dat >> 16) & (dat & 0xFFFF)) | (isr-pciisr);
  4.1039+	pcicfgw16(p, cap + (ok64 ? MSIData64 : MSIData32), dat);
  4.1040+	pcicfgw16(p, cap + MSICtrl, 1);
  4.1041+}
  4.1042+
  4.1043+void
  4.1044+pciintrdisable(int tbdf, void (*f)(Ureg*, void*), void *a)
  4.1045+{
  4.1046+	Pciisr *isr;
  4.1047+
  4.1048+	lock(&pciisrlk);
  4.1049+	for(isr = pciisr; isr < &pciisr[nelem(pciisr)]; isr++){
  4.1050+		if(isr->p != nil && isr->p->tbdf == tbdf && isr->f == f && isr->a == a){
  4.1051+			regs[MSI_INTR2_BASE + INTR_MASK_SET] = 1 << (isr-pciisr);
  4.1052+			isr->p = nil;
  4.1053+			isr->f = nil;
  4.1054+			isr->a = nil;
  4.1055+			break;
  4.1056+		}
  4.1057+	}
  4.1058+	unlock(&pciisrlk);
  4.1059+}
  4.1060+
  4.1061+static void
  4.1062+pciinterrupt(Ureg *ureg, void*)
  4.1063+{
  4.1064+	Pciisr *isr;
  4.1065+	u32int sts;
  4.1066+
  4.1067+	sts = regs[MSI_INTR2_BASE + INTR_STATUS];
  4.1068+	if(sts == 0)
  4.1069+		return;
  4.1070+	regs[MSI_INTR2_BASE + INTR_CLR] = sts;
  4.1071+	for(isr = pciisr; sts != 0 && isr < &pciisr[nelem(pciisr)]; isr++, sts>>=1){
  4.1072+		if((sts & 1) != 0 && isr->f != nil)
  4.1073+			(*isr->f)(ureg, isr->a);
  4.1074+	}
  4.1075+	regs[MISC_EOI_CTRL] = 1;
  4.1076+}
  4.1077+
  4.1078+void
  4.1079+pcilink(void)
  4.1080+{
  4.1081+	int log2dmasize = 30;	// 1GB
  4.1082+
  4.1083+	regs[RGR1_SW_INIT_1] |= 3;
  4.1084+	delay(200);
  4.1085+	regs[RGR1_SW_INIT_1] &= ~2;
  4.1086+	regs[MISC_PCIE_CTRL] &= ~5;
  4.1087+	delay(200);
  4.1088+
  4.1089+	regs[MISC_HARD_PCIE_HARD_DEBUG] &= ~0x08000000;
  4.1090+	delay(200);
  4.1091+
  4.1092+	regs[MSI_INTR2_BASE + INTR_CLR] = -1;
  4.1093+	regs[MSI_INTR2_BASE + INTR_MASK_SET] = -1;
  4.1094+
  4.1095+	regs[MISC_CPU_2_PCIE_MEM_WIN0_LO] = 0;
  4.1096+	regs[MISC_CPU_2_PCIE_MEM_WIN0_HI] = 0;
  4.1097+	regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_LIMIT] = 0;
  4.1098+	regs[MISC_CPU_2_PCIE_MEM_WIN0_BASE_HI] = 0;
  4.1099+	regs[MISC_CPU_2_PCIE_MEM_WIN0_LIMIT_HI] = 0;
  4.1100+
  4.1101+	// SCB_ACCESS_EN, CFG_READ_UR_MODE, MAX_BURST_SIZE_128, SCB0SIZE
  4.1102+	regs[MISC_MISC_CTRL] = 1<<12 | 1<<13 | 0<<20 | (log2dmasize-15)<<27;
  4.1103+
  4.1104+	regs[MISC_RC_BAR2_CONFIG_LO] = (log2dmasize-15);
  4.1105+	regs[MISC_RC_BAR2_CONFIG_HI] = 0;
  4.1106+
  4.1107+	regs[MISC_RC_BAR1_CONFIG_LO] = 0;
  4.1108+	regs[MISC_RC_BAR3_CONFIG_LO] = 0;
  4.1109+
  4.1110+	regs[MISC_MSI_BAR_CONFIG_LO] = MSI_TARGET_ADDR | 1;
  4.1111+	regs[MISC_MSI_BAR_CONFIG_HI] = MSI_TARGET_ADDR>>32;
  4.1112+	regs[MISC_MSI_DATA_CONFIG] = 0xFFF86540;
  4.1113+	intrenable(IRQpci, pciinterrupt, nil, BUSUNKNOWN, "pci");
  4.1114+
  4.1115+	// force to GEN2
  4.1116+	regs[(0xAC + 12)/4] = (regs[(0xAC + 12)/4] & ~15) | 2;	// linkcap
  4.1117+	regs[(0xAC + 48)/4] = (regs[(0xAC + 48)/4] & ~15) | 2;	// linkctl2
  4.1118+
  4.1119+	regs[RGR1_SW_INIT_1] &= ~1;
  4.1120+	delay(500);
  4.1121+
  4.1122+	if((regs[MISC_PCIE_STATUS] & 0x30) != 0x30){
  4.1123+		print("pcireset: phy link is down\n");
  4.1124+		return;
  4.1125+	}
  4.1126+
  4.1127+	regs[RC_CFG_PRIV1_ID_VAL3] = 0x060400;
  4.1128+	regs[RC_CFG_VENDOR_VENDOR_SPECIFIC_REG1] &= ~0xC;
  4.1129+	regs[MISC_HARD_PCIE_HARD_DEBUG] |= 2;
  4.1130+
  4.1131+	pcicfginit();
  4.1132+	pcihinv(nil);
  4.1133+}