diff --git a/sys/dev/nvme/nvme.c b/sys/dev/nvme/nvme.c
index 85820a23078..51844bfe5b7 100644
--- a/sys/dev/nvme/nvme.c
+++ b/sys/dev/nvme/nvme.c
@@ -262,6 +262,8 @@ nvme_attach(device_t dev)
 	uint32_t		devid;
 	uint16_t		subdevice;
 
+	bzero(ctrlr, sizeof(*ctrlr));
+
 	devid = pci_get_devid(dev);
 	subdevice = pci_get_subdevice(dev);
 	ep = pci_ids;
@@ -272,6 +274,22 @@ nvme_attach(device_t dev)
 	}
 	ctrlr->quirks = ep->quirks;
 
+	status = bus_dma_tag_create(bus_get_dma_tag(dev), /* parent */
+	    1, 0,			/* alignment, boundary */
+	    BUS_SPACE_MAXADDR,		/* lowaddr */
+	    BUS_SPACE_MAXADDR,		/* highaddr */
+	    NULL, NULL,			/* filter, filterarg */
+	    BUS_SPACE_MAXSIZE,		/* maxsize */
+	    BUS_SPACE_UNRESTRICTED,	/* nsegments */
+	    BUS_SPACE_MAXSIZE,		/* maxsegsize */
+	    0,                          /* flags */
+	    NULL, NULL,                 /* lockfunc, lockarg */
+	    &ctrlr->nvme_parent_dma_tag);
+        if (status != 0) {
+                nvme_printf(ctrlr,"Failed to create parent DMA tag!\n");
+		return (status);
+        }
+
 	status = nvme_ctrlr_construct(ctrlr, dev);
 
 	if (status != 0) {
@@ -336,6 +354,8 @@ nvme_detach (device_t dev)
 
 	nvme_ctrlr_destruct(ctrlr, dev);
 	pci_disable_busmaster(dev);
+	if (ctrlr->nvme_parent_dma_tag != NULL)
+		bus_dma_tag_destroy(ctrlr->nvme_parent_dma_tag);
 	return (0);
 }
 
diff --git a/sys/dev/nvme/nvme_ctrlr.c b/sys/dev/nvme/nvme_ctrlr.c
index 6d6e16e718e..6358f78509a 100644
--- a/sys/dev/nvme/nvme_ctrlr.c
+++ b/sys/dev/nvme/nvme_ctrlr.c
@@ -66,10 +66,13 @@ nvme_ctrlr_allocate_bar(struct nvme_controller *ctrlr)
 		nvme_printf(ctrlr, "unable to allocate pci resource\n");
 		return (ENOMEM);
 	}
-
+nvme_printf(ctrlr, "resource: %p\n", ctrlr->resource);
 	ctrlr->bus_tag = rman_get_bustag(ctrlr->resource);
+nvme_printf(ctrlr, "bus_tag: %p\n", ctrlr->bus_tag);
 	ctrlr->bus_handle = rman_get_bushandle(ctrlr->resource);
+nvme_printf(ctrlr, "bus_handle: %p\n", ctrlr->bus_handle);
 	ctrlr->regs = (struct nvme_registers *)ctrlr->bus_handle;
+nvme_printf(ctrlr, "regs: %p\n", ctrlr->regs);
 
 	/*
 	 * The NVMe spec allows for the MSI-X table to be placed behind
@@ -220,6 +223,7 @@ nvme_ctrlr_post_failed_request(struct nvme_controller *ctrlr,
     struct nvme_request *req)
 {
 
+	nvme_printf(ctrlr, "Posting failed request!\n");
 	mtx_lock(&ctrlr->lock);
 	STAILQ_INSERT_TAIL(&ctrlr->fail_req, req, stailq);
 	mtx_unlock(&ctrlr->lock);
@@ -249,8 +253,24 @@ nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
 	int ms_waited;
 	uint32_t csts;
 
+	if (cold) {
+		nvme_printf(ctrlr, "Hey! We're not ready to wait for ready yet!\n");
+/*
+		while (cold) {
+			DELAY(1000);
+		}
+		nvme_printf(ctrlr, "OK, doing the waiting now.\n");
+*/
+	}
+
 	csts = nvme_mmio_read_4(ctrlr, csts);
 
+nvme_printf(ctrlr, "Beginning wait for ready! ctrlr %p csts %x\n", ctrlr, csts);
+DELAY(1000);
+nvme_printf(ctrlr, "Delay works.\n");
+
+	
+
 	ms_waited = 0;
 	while (((csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK) != desired_val) {
 		if (ms_waited++ > ctrlr->ready_timeout_in_ms) {
@@ -261,6 +281,7 @@ nvme_ctrlr_wait_for_ready(struct nvme_controller *ctrlr, int desired_val)
 		DELAY(1000);
 		csts = nvme_mmio_read_4(ctrlr, csts);
 	}
+nvme_printf(ctrlr, "Ending wait for ready!\n");
 
 	return (0);
 }
@@ -276,6 +297,21 @@ nvme_ctrlr_disable(struct nvme_controller *ctrlr)
 	cc = nvme_mmio_read_4(ctrlr, cc);
 	csts = nvme_mmio_read_4(ctrlr, csts);
 
+	if (cc == 0xffffffff && csts == 0xffffffff) {
+		/*
+		 * It's worse than that, he's dead, Jim.
+		 */
+		nvme_printf(ctrlr, "Oh dear, the controller appears to have fallen off the bus.\n");
+		if (pcie_flr(ctrlr->dev, max(pcie_get_max_completion_timeout(ctrlr->dev) / 1000, 10), true)) {
+			nvme_printf(ctrlr, "What? That actually worked?\n");
+			cc = nvme_mmio_read_4(ctrlr, cc);
+			csts = nvme_mmio_read_4(ctrlr, csts);
+		}
+		else { 
+			panic("bad mojo");
+		}
+	}
+
 	en = (cc >> NVME_CC_REG_EN_SHIFT) & NVME_CC_REG_EN_MASK;
 	rdy = (csts >> NVME_CSTS_REG_RDY_SHIFT) & NVME_CSTS_REG_RDY_MASK;
 
@@ -405,13 +441,15 @@ nvme_ctrlr_reset(struct nvme_controller *ctrlr)
 
 	cmpset = atomic_cmpset_32(&ctrlr->is_resetting, 0, 1);
 
-	if (cmpset == 0 || ctrlr->is_failed)
+	if (cmpset == 0 || ctrlr->is_failed) {
+nvme_printf(ctrlr, "multiple nvme_ctrlr_reset!\n");
 		/*
 		 * Controller is already resetting or has failed.  Return
 		 *  immediately since there is no need to kick off another
 		 *  reset in these cases.
 		 */
 		return;
+	}
 
 	taskqueue_enqueue(ctrlr->taskqueue, &ctrlr->reset_task);
 }
@@ -830,6 +868,7 @@ nvme_ctrlr_configure_int_coalescing(struct nvme_controller *ctrlr)
 static void
 nvme_ctrlr_start(void *ctrlr_arg)
 {
+	printf("nvme_ctrlr_start!\n");
 	struct nvme_controller *ctrlr = ctrlr_arg;
 	uint32_t old_num_io_queues;
 	int i;
@@ -930,7 +969,7 @@ nvme_ctrlr_reset_task(void *arg, int pending)
 	 *
 	 * XXX - any way to guarantee the interrupt handlers have quiesced?
 	 */
-	pause("nvmereset", hz / 10);
+	pause("nvmereset", hz * 2);
 	if (status == 0)
 		nvme_ctrlr_start(ctrlr);
 	else
diff --git a/sys/dev/nvme/nvme_private.h b/sys/dev/nvme/nvme_private.h
index 19d19f1d4cc..07fd7520892 100644
--- a/sys/dev/nvme/nvme_private.h
+++ b/sys/dev/nvme/nvme_private.h
@@ -285,8 +285,7 @@ struct nvme_controller {
 	struct resource		*res;
 	void			*tag;
 
-	bus_dma_tag_t		hw_desc_tag;
-	bus_dmamap_t		hw_desc_map;
+	bus_dma_tag_t		nvme_parent_dma_tag;
 
 	/** maximum i/o size in bytes */
 	uint32_t		max_xfer_size;
diff --git a/sys/dev/nvme/nvme_qpair.c b/sys/dev/nvme/nvme_qpair.c
index 3fe691963f4..8d5ea156653 100644
--- a/sys/dev/nvme/nvme_qpair.c
+++ b/sys/dev/nvme/nvme_qpair.c
@@ -565,8 +565,12 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
 
 	mtx_init(&qpair->lock, "nvme qpair lock", NULL, MTX_DEF);
 
+nvme_printf(ctrlr, "dev %p\n", ctrlr->dev);
+nvme_printf(ctrlr, "bus dma tag %p\n", bus_get_dma_tag(ctrlr->dev));
+nvme_printf(ctrlr, "device parent %p\n", device_get_parent(ctrlr->dev));
+
 	/* Note: NVMe PRP format is restricted to 4-byte alignment. */
-	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
+	err = bus_dma_tag_create(ctrlr->nvme_parent_dma_tag,
 	    4, PAGE_SIZE, BUS_SPACE_MAXADDR,
 	    BUS_SPACE_MAXADDR, NULL, NULL, NVME_MAX_XFER_SIZE,
 	    (NVME_MAX_XFER_SIZE/PAGE_SIZE)+1, PAGE_SIZE, 0,
@@ -575,6 +579,7 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
 		nvme_printf(ctrlr, "payload tag create failed %d\n", err);
 		goto out;
 	}
+nvme_printf(ctrlr, "payload tag %p\n", qpair->dma_tag_payload);
 
 	/*
 	 * Each component must be page aligned, and individual PRP lists
@@ -588,13 +593,14 @@ nvme_qpair_construct(struct nvme_qpair *qpair, uint32_t id,
 	prpmemsz = qpair->num_trackers * prpsz;
 	allocsz = cmdsz + cplsz + prpmemsz;
 
-	err = bus_dma_tag_create(bus_get_dma_tag(ctrlr->dev),
+	err = bus_dma_tag_create(ctrlr->nvme_parent_dma_tag,
 	    PAGE_SIZE, 0, BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR, NULL, NULL,
 	    allocsz, 1, allocsz, 0, NULL, NULL, &qpair->dma_tag);
 	if (err != 0) {
 		nvme_printf(ctrlr, "tag create failed %d\n", err);
 		goto out;
 	}
+nvme_printf(ctrlr, "main tag %p\n", qpair->dma_tag);
 
 	if (bus_dmamem_alloc(qpair->dma_tag, (void **)&queuemem,
 	    BUS_DMA_NOWAIT, &qpair->queuemem_map)) {
@@ -786,6 +792,9 @@ nvme_timeout(void *arg)
 	 * controller and hope for the best.
 	 */
 	csts = nvme_mmio_read_4(ctrlr, csts);
+
+nvme_printf(ctrlr, "Timeout occurred, tracker %p qpair %p ctrlr %p csts %x\n", tr, qpair, ctrlr, csts);
+
 	cfs = (csts >> NVME_CSTS_REG_CFS_SHIFT) & NVME_CSTS_REG_CFS_MASK;
 	if (cfs == 0 && nvme_qpair_process_completions(qpair)) {
 		nvme_printf(ctrlr, "Missing interrupt\n");
diff --git a/sys/dev/ofw/ofwbus.c b/sys/dev/ofw/ofwbus.c
index 2a28a928e15..16e0796cb68 100644
--- a/sys/dev/ofw/ofwbus.c
+++ b/sys/dev/ofw/ofwbus.c
@@ -100,7 +100,7 @@ static device_method_t ofwbus_methods[] = {
 DEFINE_CLASS_1(ofwbus, ofwbus_driver, ofwbus_methods,
     sizeof(struct ofwbus_softc), simplebus_driver);
 static devclass_t ofwbus_devclass;
-EARLY_DRIVER_MODULE(ofwbus, nexus, ofwbus_driver, ofwbus_devclass, 0, 0,
+EARLY_BUS_DRIVER_MODULE(ofwbus, nexus, ofwbus_driver, ofwbus_devclass, 0, 0,
     BUS_PASS_BUS + BUS_PASS_ORDER_MIDDLE);
 MODULE_VERSION(ofwbus, 1);
 
diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 289cfddd185..bf11fe26e10 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -55,6 +55,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/uma.h>
 #include <vm/uma_dbg.h>
 
+#include <sys/kdb.h>
+
 /*
  * In FreeBSD, Mbufs and Mbuf Clusters are allocated from UMA
  * Zones.
@@ -487,24 +489,39 @@ netdump_mbuf_drain(void)
 {
 	struct mbuf *m;
 	void *item;
+printf("netdump_mbuf_drain!\n");
 
 	if (nd_zone_mbuf != NULL) {
+		printf("nd_zone_mbuf destroy %p\n", nd_zone_mbuf);
 		uma_zdestroy(nd_zone_mbuf);
 		nd_zone_mbuf = NULL;
 	}
 	if (nd_zone_clust != NULL) {
+		printf("nd_zone_clust destroy %p\n", nd_zone_clust);
 		uma_zdestroy(nd_zone_clust);
 		nd_zone_clust = NULL;
 	}
 	if (nd_zone_pack != NULL) {
+		printf("nd_zone_pack destroy %p\n", nd_zone_pack);
 		uma_zdestroy(nd_zone_pack);
 		nd_zone_pack = NULL;
 	}
 
-	while ((m = mbufq_dequeue(&nd_mbufq)) != NULL)
-		m_free(m);
-	while ((item = mbufq_dequeue(&nd_clustq)) != NULL)
-		uma_zfree(m_getzone(nd_clsize), item);
+	if (mbufq_len(&nd_mbufq) > 0) {
+		printf("mbufq destroy %p len %d\n", nd_mbufq, mbufq_len(&nd_mbufq));
+		while ((m = mbufq_dequeue(&nd_mbufq)) != NULL) {
+			printf("nd_mbufq free %p\n", m);
+                kdb_enter(KDB_WHY_BOOTFLAGS, "bleh");
+
+			m_free(m);
+		}
+	}
+	if (mbufq_len(&nd_clustq) > 0)
+		while ((item = mbufq_dequeue(&nd_clustq)) != NULL) {
+			printf("netbuf size %ld item %p\n", nd_clsize, item);
+			printf("xxx %p\n", m_getzone(nd_clsize));
+			uma_zfree(m_getzone(nd_clsize), item);
+		}
 }
 
 /*
diff --git a/sys/kern/link_elf.c b/sys/kern/link_elf.c
index c8d3d81c350..fa95822e37f 100644
--- a/sys/kern/link_elf.c
+++ b/sys/kern/link_elf.c
@@ -1559,6 +1559,7 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res)
 	 * Don't do a full lookup when the symbol is local. It may even
 	 * fail because it may not be found through the hash table.
 	 */
+#if 0
 	if (ELF_ST_BIND(sym->st_info) == STB_LOCAL) {
 		/* Force lookup failure when we have an insanity. */
 		if (sym->st_shndx == SHN_UNDEF || sym->st_value == 0) {
@@ -1568,6 +1569,7 @@ elf_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res)
 		*res = ((Elf_Addr)ef->address + sym->st_value);
 		return (0);
 	}
+#endif
 
 	/*
 	 * XXX we can avoid doing a hash table based lookup for global
diff --git a/sys/kern/link_elf_obj.c b/sys/kern/link_elf_obj.c
index 15548e6e469..21dd52227d8 100644
--- a/sys/kern/link_elf_obj.c
+++ b/sys/kern/link_elf_obj.c
@@ -1397,6 +1397,7 @@ elf_obj_lookup(linker_file_t lf, Elf_Size symidx, int deps, Elf_Addr *res)
 	/* If we get here, then it is undefined and needs a lookup. */
 	switch (ELF_ST_BIND(sym->st_info)) {
 	case STB_LOCAL:
+printf("Local but undefined\n");
 		/* Local, but undefined? huh? */
 		*res = 0;
 		return (EINVAL);
diff --git a/sys/kern/subr_bus.c b/sys/kern/subr_bus.c
index 27b00656fc2..5a707ee66e9 100644
--- a/sys/kern/subr_bus.c
+++ b/sys/kern/subr_bus.c
@@ -4962,7 +4962,7 @@ static moduledata_t root_bus_mod = {
 	root_bus_module_handler,
 	NULL
 };
-DECLARE_MODULE(rootbus, root_bus_mod, SI_SUB_DRIVERS, SI_ORDER_FIRST);
+DECLARE_MODULE(rootbus, root_bus_mod, SI_SUB_EARLY_BUS, SI_ORDER_FIRST);
 
 /**
  * @brief Automatically configure devices
diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c
index 61616088871..5a72e97fb68 100644
--- a/sys/kern/subr_kdb.c
+++ b/sys/kern/subr_kdb.c
@@ -646,6 +646,7 @@ kdb_thr_select(struct thread *thr)
 int
 kdb_trap(int type, int code, struct trapframe *tf)
 {
+	printf("@");
 #ifdef SMP
 	cpuset_t other_cpus;
 #endif
diff --git a/sys/kern/subr_rman.c b/sys/kern/subr_rman.c
index cc0bcdc8f0a..8505a64318e 100644
--- a/sys/kern/subr_rman.c
+++ b/sys/kern/subr_rman.c
@@ -126,6 +126,7 @@ int_alloc_resource(int malloc_flag)
 int
 rman_init(struct rman *rm)
 {
+	printf("RMAN %p\n", rm);
 	static int once = 0;
 
 	if (once == 0) {
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 7a3d04caf52..f93a3765a8f 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -220,6 +220,8 @@ forward_signal(struct thread *td)
 static int
 generic_stop_cpus(cpuset_t map, u_int type)
 {
+	printf("generic_stop_cpus %d\n", type);
+
 #ifdef KTR
 	char cpusetbuf[CPUSETBUFSIZ];
 #endif
diff --git a/sys/powerpc/include/intr_machdep.h b/sys/powerpc/include/intr_machdep.h
index 6ece0fa804f..7ac542537ab 100644
--- a/sys/powerpc/include/intr_machdep.h
+++ b/sys/powerpc/include/intr_machdep.h
@@ -54,7 +54,7 @@ u_int	powerpc_get_irq(uint32_t, u_int);
 void	powerpc_dispatch_intr(u_int, struct trapframe *);
 int	powerpc_enable_intr(void);
 int	powerpc_setup_intr(const char *, u_int, driver_filter_t, driver_intr_t,
-	    void *, enum intr_type, void **);
+	    void *, enum intr_type, void **, int);
 int	powerpc_teardown_intr(void *);
 int	powerpc_bind_intr(u_int irq, u_char cpu);
 int	powerpc_config_intr(int, enum intr_trigger, enum intr_polarity);
diff --git a/sys/powerpc/include/param.h b/sys/powerpc/include/param.h
index e0fef628132..90122243d39 100644
--- a/sys/powerpc/include/param.h
+++ b/sys/powerpc/include/param.h
@@ -82,8 +82,12 @@
 #endif /* SMP || KLD_MODULE */
 
 #ifndef MAXMEMDOM
+#ifdef NUMA
+#define MAXMEMDOM	8
+#else
 #define	MAXMEMDOM	1
 #endif
+#endif
 
 #define	ALIGNBYTES	_ALIGNBYTES
 #define	ALIGN(p)	_ALIGN(p)
diff --git a/sys/powerpc/include/platform.h b/sys/powerpc/include/platform.h
index 3abaa40bcf0..f3f32e8c29a 100644
--- a/sys/powerpc/include/platform.h
+++ b/sys/powerpc/include/platform.h
@@ -43,6 +43,9 @@
 struct mem_region {
 	uint64_t	mr_start;
 	uint64_t	mr_size;
+#ifdef NUMA
+	int		mr_domain;
+#endif
 };
 
 /* Documentation for these functions is in platform_if.m */
diff --git a/sys/powerpc/include/smp.h b/sys/powerpc/include/smp.h
index 1bdb8cca331..7f7e767519b 100644
--- a/sys/powerpc/include/smp.h
+++ b/sys/powerpc/include/smp.h
@@ -52,6 +52,7 @@ void	ipi_selected(cpuset_t cpus, int ipi);
 struct cpuref {
 	uintptr_t	cr_hwref;
 	u_int		cr_cpuid;
+	u_int		cr_domain;
 };
 
 void	pmap_cpu_bootstrap(int);
diff --git a/sys/powerpc/include/trap.h b/sys/powerpc/include/trap.h
index b760b33c566..542452e2c43 100644
--- a/sys/powerpc/include/trap.h
+++ b/sys/powerpc/include/trap.h
@@ -135,13 +135,19 @@
  * 2.05.
  */
 
-#define	EXC_PGM_FPENABLED	(1UL << 20)
-#define	EXC_PGM_ILLEGAL		(1UL << 19)
-#define	EXC_PGM_PRIV		(1UL << 18)
-#define	EXC_PGM_TRAP		(1UL << 17)
+#define EXC_PGM_BAD_THING	(1UL << 21) /* Bit 42 (POWER8+) */
+#define	EXC_PGM_FPENABLED	(1UL << 20) /* Bit 43 */
+#define	EXC_PGM_ILLEGAL		(1UL << 19) /* Bit 44 (<POWER7) */
+#define	EXC_PGM_PRIV		(1UL << 18) /* Bit 45 */
+#define	EXC_PGM_TRAP		(1UL << 17) /* Bit 46 */
+
+/*
+ * When SRR1 bit 47 is set, SRR0 is NOT the faulting instruction addr.
+ */
+#define EXC_NOT_FAULT_INST	(1UL << 16) /* Bit 47 */
 
 /* DTrace trap opcode. */
-#define EXC_DTRACE	0x7ffff808
+#define EXC_DTRACE	0x7ffff808 /* tw 31,31,31 */
 
 /* Magic pointer to store TOC base and other info for trap handlers on ppc64 */
 #define TRAP_GENTRAP	0x1f0
diff --git a/sys/powerpc/ofw/ofw_machdep.c b/sys/powerpc/ofw/ofw_machdep.c
index 285c010d573..2879e690e81 100644
--- a/sys/powerpc/ofw/ofw_machdep.c
+++ b/sys/powerpc/ofw/ofw_machdep.c
@@ -76,6 +76,11 @@ __FBSDID("$FreeBSD$");
 static void	*fdt;
 int		ofw_real_mode;
 
+#ifdef NUMA
+struct mem_affinity mem_info[VM_PHYSSEG_MAX + 1];
+static int vm_locality_table[MAXMEMDOM * MAXMEMDOM];
+#endif
+
 #ifdef AIM
 extern register_t ofmsr[5];
 extern void	*openfirmware_entry;
@@ -193,6 +198,11 @@ parse_ofw_memory(phandle_t node, const char *prop, struct mem_region *output)
 	i = 0;
 	j = 0;
 	while (i < sz/sizeof(cell_t)) {
+		
+#ifdef NUMA
+		OF_getencprop(node, 
+		output[j].mr_domain = 0;
+#endif
 		output[j].mr_start = OFmem[i++];
 		if (address_cells == 2) {
 			output[j].mr_start <<= 32;
diff --git a/sys/powerpc/powernv/opal.h b/sys/powerpc/powernv/opal.h
index 7fc68dee9f9..efb8279485c 100644
--- a/sys/powerpc/powernv/opal.h
+++ b/sys/powerpc/powernv/opal.h
@@ -52,6 +52,7 @@ int opal_call(uint64_t token, ...);
 #define	OPAL_PCI_CONFIG_WRITE_BYTE	16
 #define	OPAL_PCI_CONFIG_WRITE_HALF_WORD	17
 #define	OPAL_PCI_CONFIG_WRITE_WORD	18
+#define OPAL_PCI_EEH_FREEZE_STATUS	23
 #define	OPAL_PCI_EEH_FREEZE_CLEAR	26
 #define	OPAL_PCI_PHB_MMIO_ENABLE	27
 #define	OPAL_PCI_SET_PHB_MEM_WINDOW	28
@@ -134,6 +135,25 @@ enum opal_msg_type {
 	OPAL_MSG_TYPE_MAX,
 };
 
+enum opal_eeh_freeze_state {
+	OPAL_EEH_STOPPED_NOT_FROZEN		= 0,
+	OPAL_EEH_STOPPED_MMIO_FREEZE		= 1,
+	OPAL_EEH_STOPPED_DMA_FREEZE		= 2,
+	OPAL_EEH_STOPPED_MMIO_DMA_FREEZE	= 3,
+	OPAL_EEH_STOPPED_RESET			= 4,
+	OPAL_EEH_STOPPED_TEMP_UNAVAIL		= 5,
+	OPAL_EEH_STOPPED_PERM_UNAVAIL		= 6,
+};
+
+enum opal_eeh_pci_error_type {
+	OPAL_EEH_NO_ERROR		= 0,
+	OPAL_EEH_IOC_ERROR		= 1,
+	OPAL_EEH_PHB_ERROR		= 2,
+	OPAL_EEH_PE_ERROR		= 3,
+	OPAL_EEH_PE_MMIO_ERROR		= 4,
+	OPAL_EEH_PE_DMA_ERROR		= 5,
+};
+
 #define	OPAL_IPMI_MSG_FORMAT_VERSION_1	1
 
 struct opal_ipmi_msg {
diff --git a/sys/powerpc/powernv/opal_dev.c b/sys/powerpc/powernv/opal_dev.c
index 8aa832967a8..cb395ea6e2d 100644
--- a/sys/powerpc/powernv/opal_dev.c
+++ b/sys/powerpc/powernv/opal_dev.c
@@ -97,9 +97,6 @@ DRIVER_MODULE(opaldev, ofwbus, opaldev_driver, opaldev_devclass, 0, 0);
 static int
 opaldev_probe(device_t dev)
 {
-	phandle_t iparent;
-	pcell_t *irqs;
-	int i, n_irqs;
 
 	if (!ofw_bus_is_compatible(dev, "ibm,opal-v3"))
 		return (ENXIO);
@@ -108,6 +105,11 @@ opaldev_probe(device_t dev)
 
 	device_set_desc(dev, "OPAL Abstraction Firmware");
 
+//#ifndef EARLY_AP_STARTUP
+	phandle_t iparent;
+	pcell_t *irqs;
+	int i, n_irqs;
+
 	/* Manually add IRQs before attaching */
 	if (OF_hasprop(ofw_bus_get_node(dev), "opal-interrupts")) {
 		iparent = OF_finddevice("/interrupt-controller@0");
@@ -123,6 +125,7 @@ opaldev_probe(device_t dev)
 			    ofw_bus_map_intr(dev, iparent, 1, &irqs[i]), 1);
 		free(irqs, M_DEVBUF);
 	}
+//#endif
 
 
 	return (BUS_PROBE_SPECIFIC);
diff --git a/sys/powerpc/powernv/opal_pci.c b/sys/powerpc/powernv/opal_pci.c
index 99a47f1b9a6..95b36f58804 100644
--- a/sys/powerpc/powernv/opal_pci.c
+++ b/sys/powerpc/powernv/opal_pci.c
@@ -38,6 +38,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/rman.h>
 #include <sys/vmem.h>
 
+#include <sys/kdb.h>
+
 #include <dev/ofw/openfirm.h>
 #include <dev/ofw/ofw_pci.h>
 #include <dev/ofw/ofw_bus.h>
@@ -65,6 +67,8 @@ __FBSDID("$FreeBSD$");
 #define	OPAL_PCI_TCE_W			(1UL << 1)
 #define	PHB3_TCE_KILL_INVAL_ALL		(1UL << 63)
 
+static void opalpci_eeh_freeze_status(device_t);
+
 /*
  * Device interface.
  */
@@ -179,7 +183,7 @@ opalpci_probe(device_t dev)
 		return (ENXIO); 
 
 	device_set_desc(dev, "OPAL Host-PCI bridge");
-	return (BUS_PROBE_GENERIC);
+	return (BUS_PROBE_SPECIFIC);
 }
 
 static void
@@ -269,7 +273,6 @@ opalpci_attach(device_t dev)
 		return (ENXIO);
 	}
 
-#if 0
 	/*
 	 * Reset PCI IODA table
 	 */
@@ -304,7 +307,6 @@ opalpci_attach(device_t dev)
 			DELAY(1000*(err + 1)); /* Returns expected delay in ms */
 		}
 	}
-#endif
 
 	/*
 	 * Map all devices on the bus to partitionable endpoint one until
@@ -493,6 +495,7 @@ opalpci_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
 		    config_addr, reg, vtophys(&word));
 		break;
 	default:
+		printf("opal bad width\n");
 		error = OPAL_SUCCESS;
 		word = 0xffffffff;
 	}
@@ -503,11 +506,16 @@ opalpci_read_config(device_t dev, u_int bus, u_int slot, u_int func, u_int reg,
 	 *
 	 * XXX: Make this conditional on the existence of a freeze
 	 */
+
+opalpci_eeh_freeze_status(dev);
+
 	opal_call(OPAL_PCI_EEH_FREEZE_CLEAR, sc->phb_id, OPAL_PCI_DEFAULT_PE,
 	    OPAL_EEH_ACTION_CLEAR_FREEZE_ALL);
 	
-	if (error != OPAL_SUCCESS)
+	if (error != OPAL_SUCCESS) {
+		printf("EEH_FREEZE_CLEAR failure\n");
 		word = 0xffffffff;
+	}
 
 	return (word);
 }
@@ -549,6 +557,31 @@ opalpci_write_config(device_t dev, u_int bus, u_int slot, u_int func,
 	}
 }
 
+static void
+opalpci_eeh_freeze_status(device_t dev)
+{
+	struct opalpci_softc *sc;
+	uint8_t freeze_state;
+	uint16_t pci_error_type;
+	int error = OPAL_SUCCESS;
+
+        sc = device_get_softc(dev);
+
+	error = opal_call(OPAL_PCI_EEH_FREEZE_STATUS, sc->phb_id,
+	    OPAL_PCI_DEFAULT_PE, vtophys(&freeze_state),
+	    vtophys(&pci_error_type), 0);
+
+	if (error != OPAL_SUCCESS) {
+		printf("Failed to read freeze status!\n");
+		return;
+	}
+
+	if (freeze_state != OPAL_EEH_STOPPED_NOT_FROZEN) {
+		device_printf(dev, "EEH freeze detected! freeze_state %d pci_error_type %d\n", freeze_state, pci_error_type);
+		kdb_enter(KDB_WHY_UNSET, "Uh oh!");
+	}
+}
+
 static int
 opalpci_route_interrupt(device_t bus, device_t dev, int pin)
 {
diff --git a/sys/powerpc/powernv/platform_powernv.c b/sys/powerpc/powernv/platform_powernv.c
index d4934197c92..e3a92602ec8 100644
--- a/sys/powerpc/powernv/platform_powernv.c
+++ b/sys/powerpc/powernv/platform_powernv.c
@@ -311,8 +311,10 @@ powernv_cpuref_init(void)
 		if (res > 0 && strcmp(buf, "cpu") == 0) {
 			res = OF_getproplen(cpu, "ibm,ppc-interrupt-server#s");
 			if (res > 0) {
-
-
+				tmp_cpuref[tmp_cpuref_cnt].cr_domain = 0;
+#ifdef NUMA
+				/* TODO: figure this out from chip-id and ibm,associativity! */
+#endif
 				OF_getencprop(cpu, "ibm,ppc-interrupt-server#s",
 				    interrupt_servers, res);
 
diff --git a/sys/powerpc/powerpc/autoconf.c b/sys/powerpc/powerpc/autoconf.c
index ffa6316b92c..78512ca8458 100644
--- a/sys/powerpc/powerpc/autoconf.c
+++ b/sys/powerpc/powerpc/autoconf.c
@@ -52,7 +52,11 @@ static void	configure_first(void *);
 static void	configure(void *);
 static void	configure_final(void *);
 
+#ifdef EARLY_AP_STARTUP
+SYSINIT(configure1, SI_SUB_EARLY_BUS, SI_ORDER_SECOND, configure_first, NULL);
+#else
 SYSINIT(configure1, SI_SUB_CONFIGURE, SI_ORDER_FIRST, configure_first, NULL);
+#endif
 /* SI_ORDER_SECOND is hookable */
 SYSINIT(configure2, SI_SUB_CONFIGURE, SI_ORDER_THIRD, configure, NULL);
 /* SI_ORDER_MIDDLE is hookable */
@@ -64,14 +68,22 @@ SYSINIT(configure3, SI_SUB_CONFIGURE, SI_ORDER_ANY, configure_final, NULL);
 static void
 configure_first(void *dummy)
 {
-
+	printf("CONFIGURE_FIRST!\n");
+	/* Root bus has just attached, attach the nexus */
 	nexusdev = device_add_child(root_bus, "nexus", 0);
+	printf("Going to bus_pass_resource!\n");
+	bus_set_pass(BUS_PASS_RESOURCE);
+#ifdef EARLY_AP_STARTUP
+	printf("Going to bus_pass_interrupt!\n");
+	bus_set_pass(BUS_PASS_INTERRUPT);
+	printf("At bus_pass_interrupt!\n");
+#endif
 }
 
 static void
 configure(void *dummy)
 {
-
+	printf("root_bus_configure()\n");
 	root_bus_configure();
 #ifdef DEV_ISA
 	if (isa_bus_device)
@@ -83,16 +95,26 @@ static void
 configure_final(void *dummy)
 {
 
+#ifndef EARLY_AP_STARTUP
+
 	/*
 	 * Now that we're guaranteed to have a PIC driver (or we'll never
 	 * have one), program it with all the previously setup interrupts.
 	 */
+printf("Enabling interrupt controller!\n");
 	powerpc_enable_intr();
 
+printf("Enabling external interrupts!\n");
 	/* Enable external interrupts. */
 	PCPU_SET(intr_flags, PPC_INTR_ENABLE);
 	mtmsr_ee(mfmsr() | PSL_EE);
+printf("We survived!\n");
+
+#endif
 
 	cninit_finish();
+printf("Going warm state(2)!\n");
 	cold = 0;
+printf("We are in warm state(2)!\n");
+
 }
diff --git a/sys/powerpc/powerpc/busdma_machdep.c b/sys/powerpc/powerpc/busdma_machdep.c
index cc5212198da..ca6562cfe86 100644
--- a/sys/powerpc/powerpc/busdma_machdep.c
+++ b/sys/powerpc/powerpc/busdma_machdep.c
@@ -232,6 +232,7 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
 		   bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc,
 		   void *lockfuncarg, bus_dma_tag_t *dmat)
 {
+	printf("BUS_DMA_TAG create parent %p lowaddr %ld highaddr %ld", parent, lowaddr, highaddr);
 	bus_dma_tag_t newtag;
 	int error = 0;
 
@@ -248,6 +249,7 @@ bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
 
 	newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF,
 	    M_ZERO | M_NOWAIT);
+	printf(" tagaddr %p\n", newtag);
 	if (newtag == NULL) {
 		CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d",
 		    __func__, newtag, 0, error);
@@ -548,6 +550,7 @@ bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags,
 		(*mapp)->contigalloc = 1;
 	}
 	if (*vaddr == NULL) {
+printf("%s: tag %p flags 0x%x ENOMOEM\n", __func__, dmat, dmat->flags);
 		CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d",
 		    __func__, dmat, dmat->flags, ENOMEM);
 		return (ENOMEM);
diff --git a/sys/powerpc/powerpc/clock.c b/sys/powerpc/powerpc/clock.c
index c55fcda4428..443fcbc9de4 100644
--- a/sys/powerpc/powerpc/clock.c
+++ b/sys/powerpc/powerpc/clock.c
@@ -62,10 +62,13 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/param.h>
 #include <sys/systm.h>
+#include <sys/proc.h>
 #include <sys/kernel.h>
 #include <sys/bus.h>
 #include <sys/interrupt.h>
 #include <sys/pcpu.h>
+#include <sys/sched.h>
+#include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <sys/timeet.h>
 #include <sys/timetc.h>
@@ -201,6 +204,7 @@ decr_intr(struct trapframe *frame)
 void
 cpu_initclocks(void)
 {
+	printf("CPU_INITCLOCKS!\n");
 #ifdef EARLY_AP_STARTUP
 	struct thread *td;
 	int i;
diff --git a/sys/powerpc/powerpc/cpu.c b/sys/powerpc/powerpc/cpu.c
index 0b28d78d264..c03b1e5c55d 100644
--- a/sys/powerpc/powerpc/cpu.c
+++ b/sys/powerpc/powerpc/cpu.c
@@ -710,7 +710,16 @@ cpu_idle(int busy)
 			critical_enter();
 			sbt = cpu_idleclock();
 		}
+if (PCPU_GET(cpuid) == 0) {
+
+//if (!busy) {
+//  panic("Hoey! No sleeping allowed yet!\n");
+//}
+}
+else {
 		cpu_idle_hook(sbt);
+}
+
 		if (!busy) {
 			cpu_activeclock();
 			critical_exit();
diff --git a/sys/powerpc/powerpc/intr_machdep.c b/sys/powerpc/powerpc/intr_machdep.c
index 190b6203508..4fa91c435c8 100644
--- a/sys/powerpc/powerpc/intr_machdep.c
+++ b/sys/powerpc/powerpc/intr_machdep.c
@@ -101,6 +101,7 @@ struct powerpc_intr {
 	u_int	intline;
 	u_int	vector;
 	u_int	cntindex;
+	int	domain;
 	cpuset_t cpu;
 	enum intr_trigger trig;
 	enum intr_polarity pol;
@@ -187,10 +188,23 @@ intr_init_sources(void *arg __unused)
  */
 SYSINIT(intr_init_sources, SI_SUB_KLD, SI_ORDER_ANY, intr_init_sources, NULL);
 
+#ifdef EARLY_AP_STARTUP
+
+static void
+smp_intr_init(void *dummy __unused)
+{
+	//intr_init_cpus(); /* FIX */
+	return;
+}
+SYSINIT(smp_intr_init, SI_SUB_SMP, SI_ORDER_SECOND, smp_intr_init, NULL);
+#else
 #ifdef SMP
 static void
 smp_intr_init(void *dummy __unused)
 {
+	printf("SMP_INTR_INIT!\n");
+	printf("nvectors %d", nvectors);
+
 	struct powerpc_intr *i;
 	int vector;
 
@@ -202,6 +216,76 @@ smp_intr_init(void *dummy __unused)
 }
 SYSINIT(smp_intr_init, SI_SUB_SMP, SI_ORDER_ANY, smp_intr_init, NULL);
 #endif
+#endif
+
+
+#ifdef EARLY_AP_STARTUP
+
+static struct powerpc_intr * intr_lookup(u_int irq);
+
+static void
+smp_intr_ipi_init(void *dummy __unused)
+{
+	struct powerpc_intr *i;
+	int error;
+
+	int n;
+	printf("IPI startup! Hold onto yer butts!\n");
+	if (npics == 0)
+		panic("no PIC detected\n");
+
+	if (root_pic == NULL) {
+		panic("Which pic is which? %p\n", piclist);
+		root_pic = piclist[0].dev;
+	}
+
+printf("Enabling interrupt controller!\n");
+        powerpc_enable_intr();
+
+printf("Enabling external interrupts!\n");
+        /* Enable external interrupts. */
+        mtmsr(mfmsr() | PSL_EE);
+printf("We survived!\n");
+
+printf("Going warm state!\n");
+        cold = 0;
+printf("We are in warm state!\n");
+
+	DELAY(1000);
+
+	/* Install an IPI handler. */
+	if (mp_ncpus > 1) {
+		for (n = 0; n < npics; n++) {
+			if (piclist[n].dev != root_pic)
+				continue;
+
+			KASSERT(piclist[n].ipis != 0,
+			    ("%s: SMP root PIC does not supply any IPIs",
+			    __func__));
+/* TODO: fix domain */
+			error = powerpc_setup_intr("IPI",
+			    MAP_IRQ(piclist[n].node, piclist[n].irqs),
+			    powerpc_ipi_handler, NULL, NULL,
+			    INTR_TYPE_MISC | INTR_EXCL, &ipi_cookie, 0);
+			if (error) {
+				panic("unable to setup IPI handler\n");
+			}
+
+			/*
+			 * Some subterfuge: disable late EOI and mark this
+			 * as an IPI to the dispatch layer.
+			 */
+			i = intr_lookup(MAP_IRQ(piclist[n].node,
+			    piclist[n].irqs));
+printf("intr_lookup 1\n");
+			i->event->ie_post_filter = NULL;
+			i->ipi = 1;
+		}
+	}
+}
+SYSINIT(smp_intr_ipi_init, SI_SUB_SMP, SI_ORDER_FIRST, smp_intr_ipi_init, NULL);
+#endif
+
 
 void
 intrcnt_add(const char *name, u_long **countp)
@@ -231,6 +315,7 @@ intr_lookup(u_int irq)
 			return (i);
 		}
 	}
+printf("Allocating IRQ %d\n", irq);
 
 	i = malloc(sizeof(*i), M_INTR, M_NOWAIT);
 	if (i == NULL) {
@@ -246,6 +331,7 @@ intr_lookup(u_int irq)
 	i->pic = NULL;
 	i->vector = -1;
 	i->fwcode = 0;
+	i->domain = 0;
 	i->ipi = 0;
 
 #ifdef SMP
@@ -337,6 +423,12 @@ static int
 powerpc_assign_intr_cpu(void *arg, int cpu)
 {
 #ifdef SMP
+
+#ifdef EARLY_MP_STARTUP
+	MPASS(mp_ncpus == 1 || smp_started);
+#endif
+
+	printf("intr assign cpu %d\n", cpu);
 	struct powerpc_intr *i = arg;
 
 	if (cpu == NOCPU)
@@ -344,6 +436,10 @@ powerpc_assign_intr_cpu(void *arg, int cpu)
 	else
 		CPU_SETOF(cpu, &i->cpu);
 
+	if (cold) {
+		printf("Interrup assigned too early!\n");
+	}
+
 	if (!cold && i->pic != NULL && i->pic == root_pic)
 		PIC_BIND(i->pic, i->intline, i->cpu);
 
@@ -357,6 +453,7 @@ u_int
 powerpc_register_pic(device_t dev, uint32_t node, u_int irqs, u_int ipis,
     u_int atpic)
 {
+	printf("POWERPC_REGISTER_PIC\n");
 	struct pic *p;
 	u_int irq;
 	int idx;
@@ -437,9 +534,10 @@ powerpc_get_irq(uint32_t node, u_int pin)
 int
 powerpc_enable_intr(void)
 {
+	printf("powerpc_enable_intr\n");
 	struct powerpc_intr *i;
 	int error, vector;
-#ifdef SMP
+#if defined(SMP) && !defined(EARLY_AP_STARTUP)
 	int n;
 #endif
 
@@ -449,7 +547,7 @@ powerpc_enable_intr(void)
 	if (root_pic == NULL)
 		root_pic = piclist[0].dev;
 
-#ifdef SMP
+#if defined(SMP) && !defined(EARLY_AP_STARTUP)
 	/* Install an IPI handler. */
 	if (mp_ncpus > 1) {
 		for (n = 0; n < npics; n++) {
@@ -474,6 +572,7 @@ powerpc_enable_intr(void)
 			 */
 			i = intr_lookup(MAP_IRQ(piclist[n].node,
 			    piclist[n].irqs));
+printf("intr_lookup 2\n");
 			i->event->ie_post_filter = NULL;
 			i->ipi = 1;
 		}
@@ -505,12 +604,16 @@ powerpc_enable_intr(void)
 
 int
 powerpc_setup_intr(const char *name, u_int irq, driver_filter_t filter,
-    driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep)
+    driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep,
+    int domain)
 {
+	printf("powerpc_setup_intr %s irq %d\n", name, irq);
 	struct powerpc_intr *i;
 	int error, enable = 0;
 
 	i = intr_lookup(irq);
+printf("intr_lookup 3\n");
+
 	if (i == NULL)
 		return (ENOMEM);
 
@@ -518,10 +621,14 @@ powerpc_setup_intr(const char *name, u_int irq, driver_filter_t filter,
 		error = intr_event_create(&i->event, (void *)i, 0, irq,
 		    powerpc_intr_pre_ithread, powerpc_intr_post_ithread,
 		    powerpc_intr_eoi, powerpc_assign_intr_cpu, "irq%u:", irq);
-		if (error)
+		if (error) {
+			printf("intr_event_create error!\n");
 			return (error);
+		}
 
 		enable = 1;
+
+		i->domain = domain;
 	}
 
 	error = intr_event_add_handler(i->event, name, filter, handler, arg,
@@ -531,7 +638,9 @@ powerpc_setup_intr(const char *name, u_int irq, driver_filter_t filter,
 	intrcnt_setname(i->event->ie_fullname, i->cntindex);
 	mtx_unlock(&intr_table_lock);
 
+//#ifndef EARLY_AP_STARTUP
 	if (!cold) {
+//#endif
 		error = powerpc_map_irq(i);
 
 		if (!error) {
@@ -543,13 +652,21 @@ powerpc_setup_intr(const char *name, u_int irq, driver_filter_t filter,
 			    i->pol != INTR_POLARITY_CONFORM)
 				PIC_CONFIG(i->pic, i->intline, i->trig, i->pol);
 
+			if (enable) {
+				printf("Enable!\n");
+				PIC_ENABLE(i->pic, i->intline, i->vector);
+			}
+
 			if (i->pic == root_pic)
 				PIC_BIND(i->pic, i->intline, i->cpu);
 
-			if (enable)
-				PIC_ENABLE(i->pic, i->intline, i->vector);
 		}
+//#ifndef EARLY_AP_STARTUP
+	}
+	else {
+		printf("COLD INTERRUPT!\n");
 	}
+//#endif
 	return (error);
 }
 
@@ -567,6 +684,8 @@ powerpc_bind_intr(u_int irq, u_char cpu)
 	struct powerpc_intr *i;
 
 	i = intr_lookup(irq);
+printf("intr_lookup 4\n");
+
 	if (i == NULL)
 		return (ENOMEM);
 
@@ -580,6 +699,8 @@ powerpc_fw_config_intr(int irq, int sense_code)
 	struct powerpc_intr *i;
 
 	i = intr_lookup(irq);
+printf("intr_lookup 5\n");
+
 	if (i == NULL)
 		return (ENOMEM);
 
@@ -602,6 +723,8 @@ powerpc_config_intr(int irq, enum intr_trigger trig, enum intr_polarity pol)
 	struct powerpc_intr *i;
 
 	i = intr_lookup(irq);
+printf("intr_lookup 6\n");
+
 	if (i == NULL)
 		return (ENOMEM);
 
@@ -660,6 +783,7 @@ powerpc_intr_mask(u_int irq)
 	struct powerpc_intr *i;
 
 	i = intr_lookup(irq);
+
 	if (i == NULL || i->pic == NULL)
 		return;
 
diff --git a/sys/powerpc/powerpc/mp_machdep.c b/sys/powerpc/powerpc/mp_machdep.c
index 23a943c8865..2445b12a460 100644
--- a/sys/powerpc/powerpc/mp_machdep.c
+++ b/sys/powerpc/powerpc/mp_machdep.c
@@ -260,6 +260,7 @@ cpu_mp_unleash(void *dummy)
 	
 	/* Let APs continue */
 	atomic_store_rel_int(&ap_letgo, 1);
+	printf("!");
 
 	platform_smp_timebase_sync(ap_timebase, 0);
 
@@ -274,8 +275,10 @@ cpu_mp_unleash(void *dummy)
 		    mp_ncpus, cpus, smp_cpus);
 	}
 
-	if (smp_cpus > 1)
+	if (smp_cpus > 1) {
+		printf("UNLEASH THE HOUNDS!\n");
 		atomic_store_rel_int(&smp_started, 1);
+	}
 
 	/* Let the APs get into the scheduler */
 	DELAY(1000);
diff --git a/sys/powerpc/powerpc/nexus.c b/sys/powerpc/powerpc/nexus.c
index f5a3da8eb27..de3e8f115bf 100644
--- a/sys/powerpc/powerpc/nexus.c
+++ b/sys/powerpc/powerpc/nexus.c
@@ -99,7 +99,7 @@ static device_method_t nexus_methods[] = {
 static devclass_t nexus_devclass;
 
 DEFINE_CLASS_0(nexus, nexus_driver, nexus_methods, 1);
-EARLY_DRIVER_MODULE(nexus, root, nexus_driver, nexus_devclass, 0, 0,
+EARLY_BUS_DRIVER_MODULE(nexus, root, nexus_driver, nexus_devclass, 0, 0,
     BUS_PASS_BUS);
 MODULE_VERSION(nexus, 1);
 
@@ -107,7 +107,7 @@ static int
 nexus_probe(device_t dev)
 {
         
-	device_quiet(dev);	/* suppress attach message for neatness */
+	//device_quiet(dev);	/* suppress attach message for neatness */
 
 	return (BUS_PROBE_DEFAULT);
 }
@@ -127,11 +127,15 @@ nexus_setup_intr(device_t bus __unused, device_t child, struct resource *r,
     int flags, driver_filter_t *filt, driver_intr_t *intr, void *arg,
     void **cookiep)
 {
-	int error;
+	int error, domain;
 
 	if (r == NULL)
 		panic("%s: NULL interrupt resource!", __func__);
 
+	if (cookiep != NULL) {
+		printf("cookie not null\n");
+	}
+
 	if ((rman_get_flags(r) & RF_SHAREABLE) == 0)
 		flags |= INTR_EXCL;
 
@@ -140,8 +144,10 @@ nexus_setup_intr(device_t bus __unused, device_t child, struct resource *r,
 	if (error)
 		return (error);
 
+	if (bus_get_domain(child, &domain) != 0)
+		domain = 0;
 	error = powerpc_setup_intr(device_get_nameunit(child),
-	    rman_get_start(r), filt, intr, arg, flags, cookiep);
+	    rman_get_start(r), filt, intr, arg, flags, cookiep, domain);
 
 	return (error);
 }
@@ -169,6 +175,7 @@ static int
 nexus_bind_intr(device_t bus __unused, device_t child __unused,
     struct resource *r, int cpu)
 {
+	printf("nexus_bind_intr\n");
 
 	return (powerpc_bind_intr(rman_get_start(r), cpu));
 }
diff --git a/sys/powerpc/powerpc/trap.c b/sys/powerpc/powerpc/trap.c
index 3939055afc7..4ea191f7d7a 100644
--- a/sys/powerpc/powerpc/trap.c
+++ b/sys/powerpc/powerpc/trap.c
@@ -358,6 +358,7 @@ trap(struct trapframe *frame)
 			if (frame_is_trap_inst(frame)) {
 #ifdef KDTRACE_HOOKS
 				inst = fuword32((const void *)frame->srr0);
+				/* FASTTRAP_INSTR */
 				if (inst == 0x0FFFDDDD &&
 				    dtrace_pid_probe_ptr != NULL) {
 					(*dtrace_pid_probe_ptr)(frame);
diff --git a/sys/powerpc/pseries/xics.c b/sys/powerpc/pseries/xics.c
index 9a760d59065..ecb9c939776 100644
--- a/sys/powerpc/pseries/xics.c
+++ b/sys/powerpc/pseries/xics.c
@@ -155,8 +155,8 @@ static uint32_t cpu_xirr[MAXCPU];
 static devclass_t xicp_devclass;
 static devclass_t xics_devclass;
 
-EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0,
-    BUS_PASS_INTERRUPT-1);
+EARLY_BUS_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0,
+    BUS_PASS_RESOURCE);
 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, xics_devclass, 0, 0,
     BUS_PASS_INTERRUPT);
 
@@ -187,7 +187,7 @@ xicp_probe(device_t dev)
 		return (ENXIO);
 
 	device_set_desc(dev, "External Interrupt Presentation Controller");
-	return (BUS_PROBE_GENERIC);
+	return (BUS_PROBE_SPECIFIC);
 }
 
 static int
@@ -205,6 +205,7 @@ xics_probe(device_t dev)
 static int
 xicp_attach(device_t dev)
 {
+	printf("xicp_attach!\n");
 	struct xicp_softc *sc = device_get_softc(dev);
 	phandle_t phandle = ofw_bus_get_node(dev);
 
@@ -236,6 +237,7 @@ xicp_attach(device_t dev)
 			 */
 			sc->xics_emu = true;
 			opal_call(OPAL_XIVE_RESET, XIVE_XICS_MODE_EMU);
+printf("XICS IN EMULATION MODE!\n");
 #endif
 	} else {
 		sc->cpu_range[0] = 0;
@@ -334,7 +336,9 @@ xicp_bind(device_t dev, u_int irq, cpuset_t cpumask)
 			break;
 		}
 	}
-	KASSERT(i < sc->nintvecs, ("Binding non-configured interrupt"));
+
+		printf("BIND: interrupt %d to cpu %d \n", irq, cpu); 
+	KASSERT(i < sc->nintvecs, ("Binding non-configured interrupt %d i=%d ncpus %d\n", irq, i, ncpus));
 
 	if (rtas_exists())
 		error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
@@ -353,8 +357,10 @@ xicp_dispatch(device_t dev, struct trapframe *tf)
 {
 	struct xicp_softc *sc;
 	struct resource *regs = NULL;
-	uint64_t xirr, junk;
-	int i;
+	uint64_t xirr, junk, rawxirr;
+	int i, mode;
+
+	rawxirr = 0xd0d0d0d0;
 
 	sc = device_get_softc(dev);
 #ifdef POWERNV
@@ -369,15 +375,21 @@ xicp_dispatch(device_t dev, struct trapframe *tf)
 		/* Return value in R4, use the PFT call */
 		if (regs) {
 			xirr = bus_read_4(regs, 4);
+			rawxirr = xirr;
+			mode = 1;
 #ifdef POWERNV
 		} else if (sc->xics_emu) {
 			opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)],
 			    false);
 			xirr = cpu_xirr[PCPU_GET(cpuid)];
+			rawxirr = xirr;
+			mode = 2;
 #endif
 		} else {
 			/* Return value in R4, use the PFT call */
 			phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk);
+			rawxirr = xirr;
+			mode = 3;
 		}
 		xirr &= 0x00ffffff;
 
@@ -400,20 +412,23 @@ xicp_dispatch(device_t dev, struct trapframe *tf)
 				    0xff);
 		}
 
-		/* XXX: super inefficient */
-		for (i = 0; i < sc->nintvecs; i++) {
-			if (sc->intvecs[i].irq == xirr)
-				break;
-		}
+		//if (__predict_true(sc->nintvecs)) {
+			/* XXX: super inefficient */
+			for (i = 0; i < sc->nintvecs; i++) {
+				if (sc->intvecs[i].irq == xirr)
+					break;
+			}
 
-		KASSERT(i < sc->nintvecs, ("Unmapped XIRR"));
-		powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
+			KASSERT(i < sc->nintvecs, ("Unmapped XIRR %d %d xirr %lx raw: %lx mode %d\n", i, sc->nintvecs, xirr, rawxirr, mode));
+			powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
+		//}
 	}
 }
 
 static void
 xicp_enable(device_t dev, u_int irq, u_int vector)
 {
+	device_printf(dev, "XICP enable irq %d vector %d\n", irq, vector);
 	struct xicp_softc *sc;
 	cell_t status, cpu;
 
@@ -434,8 +449,10 @@ xicp_enable(device_t dev, u_int irq, u_int vector)
 	mtx_unlock(&sc->sc_mtx);
 
 	/* IPIs are also enabled */
-	if (irq == MAX_XICP_IRQS)
+	if (irq == MAX_XICP_IRQS) {
+		printf("xicp_enable IPI\n");
 		return;
+	}
 
 	if (rtas_exists()) {
 		rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
@@ -557,6 +574,7 @@ xicp_unmask(device_t dev, u_int irq)
 void
 xicp_smp_cpu_startup(void)
 {
+	printf("xicp_smp_cpu_startup()\n");
 	struct xicp_softc *sc;
 
 	if (mfmsr() & PSL_HV) {
diff --git a/sys/sys/bus.h b/sys/sys/bus.h
index e565002760d..3ae0f8a1e0a 100644
--- a/sys/sys/bus.h
+++ b/sys/sys/bus.h
@@ -755,7 +755,7 @@ struct driver_module_data {
 };
 
 #define	EARLY_DRIVER_MODULE_ORDERED(name, busname, driver, devclass,	\
-    evh, arg, order, pass)						\
+    evh, arg, sub, order, pass)						\
 									\
 static struct driver_module_data name##_##busname##_driver_mod = {	\
 	evh, arg,							\
@@ -771,16 +771,20 @@ static moduledata_t name##_##busname##_mod = {				\
 	&name##_##busname##_driver_mod					\
 };									\
 DECLARE_MODULE(name##_##busname, name##_##busname##_mod,		\
-	       SI_SUB_DRIVERS, order)
+	       sub, order)
 
 #define	EARLY_DRIVER_MODULE(name, busname, driver, devclass, evh, arg, pass) \
 	EARLY_DRIVER_MODULE_ORDERED(name, busname, driver, devclass,	\
-	    evh, arg, SI_ORDER_MIDDLE, pass)
+	    evh, arg, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, pass)
+
+#define EARLY_BUS_DRIVER_MODULE(name, busname, driver, devclass, evh, arg, pass) \
+	EARLY_DRIVER_MODULE_ORDERED(name, busname, driver, devclass,	\
+	    evh, arg, SI_SUB_EARLY_BUS, SI_ORDER_MIDDLE, pass)
 
 #define	DRIVER_MODULE_ORDERED(name, busname, driver, devclass, evh, arg,\
     order)								\
 	EARLY_DRIVER_MODULE_ORDERED(name, busname, driver, devclass,	\
-	    evh, arg, order, BUS_PASS_DEFAULT)
+	    evh, arg, SI_SUB_DRIVERS, order, BUS_PASS_DEFAULT)
 
 #define	DRIVER_MODULE(name, busname, driver, devclass, evh, arg)	\
 	EARLY_DRIVER_MODULE(name, busname, driver, devclass, evh, arg,	\
diff --git a/sys/sys/cdefs.h b/sys/sys/cdefs.h
index a2625a1ed5b..0ef88bd3a29 100644
--- a/sys/sys/cdefs.h
+++ b/sys/sys/cdefs.h
@@ -774,8 +774,8 @@
 #endif
 #endif /* __STDC_WANT_LIB_EXT1__ */
 
-#if defined(__mips) || defined(__powerpc64__) || defined(__riscv)
-#define	__NO_TLS 1
+#if defined(__mips) || defined(__riscv) || (defined(__GNUC__) && !__GNUC_PREREQ__(4, 9))
+#define __NO_TLS 1
 #endif
 
 /*
diff --git a/sys/sys/kernel.h b/sys/sys/kernel.h
index a9bfaa9ef7c..d89e8f293c9 100644
--- a/sys/sys/kernel.h
+++ b/sys/sys/kernel.h
@@ -124,16 +124,22 @@ enum sysinit_sub_id {
 	SI_SUB_MBUF		= 0x2700000,	/* mbuf subsystem */
 	SI_SUB_INTR		= 0x2800000,	/* interrupt threads */
 	SI_SUB_TASKQ		= 0x2880000,	/* task queues */
+//	SI_SUB_DEVFS		= 0x28A0000,	/* devfs ready for devices */
 #ifdef EARLY_AP_STARTUP
+	SI_SUB_EARLY_BUS	= 0x28B0000,	/* Attach root bus and interrupt controller */
 	SI_SUB_SMP		= 0x2900000,	/* start the APs*/
 #endif
 	SI_SUB_SOFTINTR		= 0x2A00000,	/* start soft interrupt thread */
+// temp
 	SI_SUB_DEVFS		= 0x2F00000,	/* devfs ready for devices */
 	SI_SUB_INIT_IF		= 0x3000000,	/* prep for net interfaces */
 	SI_SUB_NETGRAPH		= 0x3010000,	/* Let Netgraph initialize */
 	SI_SUB_DTRACE		= 0x3020000,	/* DTrace subsystem */
 	SI_SUB_DTRACE_PROVIDER	= 0x3048000,	/* DTrace providers */
 	SI_SUB_DTRACE_ANON	= 0x308C000,	/* DTrace anon enabling */
+#ifndef EARLY_AP_STARTUP
+	SI_SUB_EARLY_BUS	= 0x30C0000,	/* Attach root bus and interrupt controller */
+#endif
 	SI_SUB_DRIVERS		= 0x3100000,	/* Let Drivers initialize */
 	SI_SUB_CONFIGURE	= 0x3800000,	/* Configure devices */
 	SI_SUB_VFS		= 0x4000000,	/* virtual filesystem*/