Below you'll find a patch that adds vt-d support
(https://software.intel.com/en-us/blogs/2009/06/25/understanding-vt-d-intel-virtualization-technology-for-directed-io).
Being able to restrict where devices perform DMA is a pretty big security
benefit. This code works on several machines but isn't quite completed yet.
For example it doesn't handle drm correctly yet (it bypasses it for now). It
also needs a lot of testing on different hardware to determine its actual
state. The time has however come to share this with the community to gauge
interest and getting some testing. Suspend/resume is also only working on some
machines and needs more testing. I think the acpidmar devices need to be one
of the
last devices suspended/first resumed.
This code was developed under contract from Company 0 for the Bitrig project.
Code was developed on OpenBSD because that is where I have most experience. I
sincerely hope this code can be imported into both trees.
Index: arch/amd64/conf/GENERIC
===================================================================
RCS file: /cvs/src/sys/arch/amd64/conf/GENERIC,v
retrieving revision 1.391
diff -u -p -u -p -r1.391 GENERIC
--- arch/amd64/conf/GENERIC 21 Jul 2015 03:38:22 -0000 1.391
+++ arch/amd64/conf/GENERIC 22 Jul 2015 20:24:49 -0000
@@ -52,6 +52,7 @@ acpiec* at acpi?
acpiprt* at acpi?
acpitz* at acpi?
acpimadt0 at acpi?
+acpidmar0 at acpi?
acpimcfg* at acpi?
acpiasus* at acpi?
acpisony* at acpi?
Index: arch/amd64/include/pci_machdep.h
===================================================================
RCS file: /cvs/src/sys/arch/amd64/include/pci_machdep.h,v
retrieving revision 1.23
diff -u -p -u -p -r1.23 pci_machdep.h
--- arch/amd64/include/pci_machdep.h 17 Jul 2015 22:42:09 -0000 1.23
+++ arch/amd64/include/pci_machdep.h 22 Jul 2015 20:24:49 -0000
@@ -90,7 +90,8 @@ void *pci_intr_establish(pci_chipset_ta
void pci_intr_disestablish(pci_chipset_tag_t, void *);
void pci_decompose_tag(pci_chipset_tag_t, pcitag_t,
int *, int *, int *);
-#define pci_probe_device_hook(c, a) (0)
+int pci_probe_device_hook(pci_chipset_tag_t,
+ struct pci_attach_args *);
void pci_dev_postattach(struct device *, struct pci_attach_args *);
Index: arch/amd64/pci/pci_machdep.c
===================================================================
RCS file: /cvs/src/sys/arch/amd64/pci/pci_machdep.c,v
retrieving revision 1.62
diff -u -p -u -p -r1.62 pci_machdep.c
--- arch/amd64/pci/pci_machdep.c 14 Mar 2015 03:38:46 -0000 1.62
+++ arch/amd64/pci/pci_machdep.c 22 Jul 2015 20:24:49 -0000
@@ -147,6 +147,15 @@ struct bus_dma_tag pci_bus_dma_tag = {
_bus_dmamem_mmap,
};
+extern void acpidmar_pci_hook(pci_chipset_tag_t, struct pci_attach_args *);
+
+int
+pci_probe_device_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
+{
+ acpidmar_pci_hook(pc, pa);
+ return (0);
+}
+
void
pci_attach_hook(struct device *parent, struct device *self,
struct pcibus_attach_args *pba)
Index: dev/acpi/acpidmar.c
===================================================================
RCS file: dev/acpi/acpidmar.c
diff -N dev/acpi/acpidmar.c
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ dev/acpi/acpidmar.c 22 Jul 2015 20:25:36 -0000
@@ -0,0 +1,1872 @@
+/*
+ * Copyright (c) 2015 Jordan Hargrave <jordan_hargr...@hotmail.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/device.h>
+#include <sys/malloc.h>
+#include <sys/queue.h>
+#include <sys/types.h>
+#include <sys/mbuf.h>
+#include <sys/proc.h>
+
+#include <uvm/uvm_extern.h>
+
+#include <machine/apicvar.h>
+#include <machine/cpuvar.h>
+#include <machine/bus.h>
+
+#include <dev/acpi/acpireg.h>
+#include <dev/acpi/acpivar.h>
+#include <dev/acpi/acpidev.h>
+#include <dev/acpi/amltypes.h>
+#include <dev/acpi/dsdt.h>
+
+#include <uvm/uvm_extern.h>
+
+#include <machine/i8259.h>
+#include <machine/i82093reg.h>
+#include <machine/i82093var.h>
+#include <machine/i82489reg.h>
+#include <machine/i82489var.h>
+
+#include <machine/mpbiosvar.h>
+
+#include <dev/pci/pcireg.h>
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcidevs.h>
+#include <dev/pci/ppbreg.h>
+
+#include "ioapic.h"
+
+#include "acpidmar.h"
+
+#define dprintf(x...)
+
+int intel_iommu_gfx_mapped = 0;
+
+/* Page Table Entry per domain */
+struct iommu_softc;
+
+static inline int mksid(int b, int d, int f)
+{
+ return (b << 8) + (d << 3) + f;
+}
+
+static inline int
+sid_devfn(int sid)
+{
+ return sid & 0xff;
+}
+
+static inline int sid_bus(int sid)
+{
+ return (sid>> 8) & 0xff;
+}
+
+static inline int sid_dev(int sid)
+{
+ return (sid>> 3) & 0x1f;
+}
+
+static inline int sid_fun(int sid)
+{
+ return (sid>> 0) & 0x7;
+}
+
+struct domain_dev {
+ int sid;
+ TAILQ_ENTRY(domain_dev) link;
+};
+
+struct domain {
+ struct iommu_softc *iommu;
+ int did;
+ int gaw;
+ struct pte_entry *pte;
+ paddr_t ptep;
+ struct bus_dma_tag dmat;
+ int flag;
+
+ char exname[32];
+ struct extent *iovamap;
+ TAILQ_HEAD(,domain_dev) devices;
+ TAILQ_ENTRY(domain) link;
+};
+
+struct dmar_devlist {
+ int type;
+ int bus;
+ int ndp;
+ struct acpidmar_devpath *dp;
+ TAILQ_ENTRY(dmar_devlist) link;
+};
+
+TAILQ_HEAD(devlist_head, dmar_devlist);
+
+struct rmrr_softc {
+ TAILQ_ENTRY(rmrr_softc) link;
+ struct devlist_head devices;
+ int segment;
+ uint64_t start;
+ uint64_t end;
+};
+
+struct atsr_softc {
+ TAILQ_ENTRY(atsr_softc) link;
+ struct devlist_head devices;
+ int segment;
+ int flags;
+};
+
+struct iommu_pic {
+ struct pic pic;
+ struct iommu_softc *iommu;
+};
+
+#define IOMMU_FLAGS_CATCHALL 0x1
+#define IOMMU_FLAGS_BAD 0x2
+#define IOMMU_FLAGS_SUSPEND 0x4
+
+struct iommu_softc {
+ TAILQ_ENTRY(iommu_softc)link;
+ struct devlist_head devices;
+ int id;
+ int flags;
+ int segment;
+
+ struct mutex reg_lock;
+
+ bus_space_tag_t iot;
+ bus_space_handle_t ioh;
+
+ uint64_t cap;
+ uint64_t ecap;
+ uint32_t gcmd;
+
+ int mgaw;
+ int agaw;
+ int ndoms;
+
+ struct root_entry *root;
+ struct context_entry *ctx[256];
+
+ void *intr;
+ struct iommu_pic pic;
+ int fedata;
+ uint64_t feaddr;
+ uint64_t rtaddr;
+
+ // Queued Invalidation
+ int qi_head;
+ int qi_tail;
+ paddr_t qip;
+ struct qi_entry *qi;
+
+ struct domain *unity;
+ TAILQ_HEAD(,domain) domains;
+};
+
+static inline int iommu_bad(struct iommu_softc *sc)
+{
+ return (sc->flags & IOMMU_FLAGS_BAD);
+}
+
+static inline int iommu_enabled(struct iommu_softc *sc)
+{
+ return (sc->gcmd & GCMD_TE);
+}
+
+struct acpidmar_softc {
+ struct device sc_dev;
+
+ pci_chipset_tag_t sc_pc;
+ bus_space_tag_t sc_memt;
+ int sc_haw;
+ int sc_flags;
+
+ TAILQ_HEAD(,iommu_softc)sc_drhds;
+ TAILQ_HEAD(,rmrr_softc) sc_rmrrs;
+ TAILQ_HEAD(,atsr_softc) sc_atsrs;
+};
+
+int acpidmar_activate(struct device *, int);
+int acpidmar_match(struct device *, void *, void *);
+void acpidmar_attach(struct device *, struct device *, void *);
+struct domain *acpidmar_pci_attach(struct acpidmar_softc *, int, int);
+
+struct cfattach acpidmar_ca = {
+ sizeof(struct acpidmar_softc), acpidmar_match, acpidmar_attach,
+ NULL, acpidmar_activate
+};
+
+struct cfdriver acpidmar_cd = {
+ NULL, "acpidmar", DV_DULL
+};
+
+struct acpidmar_softc *acpidmar_sc;
+int acpidmar_intr(void *);
+
+#define DID_UNITY 0x1
+
+struct domain *domain_create(struct iommu_softc *, int);
+struct domain *domain_lookup(struct acpidmar_softc *, int, int);
+
+void domain_unload_map(struct domain *, bus_dmamap_t);
+void domain_load_map(struct domain *, bus_dmamap_t, int, const char *);
+
+void domain_map_page(struct domain *, vaddr_t, paddr_t, int);
+void domain_map_pthru(struct domain *, paddr_t, paddr_t);
+
+void acpidmar_pci_hook(pci_chipset_tag_t, struct pci_attach_args *);
+void acpidmar_parse_devscope(union acpidmar_entry *, int, int,
+ struct devlist_head *);
+int acpidmar_match_devscope(struct devlist_head *, pci_chipset_tag_t, int);
+
+void acpidmar_init(struct acpidmar_softc *, struct acpi_dmar *);
+void acpidmar_drhd(struct acpidmar_softc *, union acpidmar_entry *);
+void acpidmar_rmrr(struct acpidmar_softc *, union acpidmar_entry *);
+void acpidmar_atsr(struct acpidmar_softc *, union acpidmar_entry *);
+
+void *acpidmar_intr_establish(void *, int, int (*)(void *), void *,
+ const char *);
+
+void iommu_writel(struct iommu_softc *, int, uint32_t);
+void iommu_writeq(struct iommu_softc *, int, uint64_t);
+uint64_t iommu_readq(struct iommu_softc *, int);
+uint32_t iommu_readl(struct iommu_softc *, int);
+void iommu_showfault(struct iommu_softc *, int,
+ struct fault_entry *);
+void iommu_showcfg(struct iommu_softc *);
+
+int iommu_init(struct acpidmar_softc *, struct iommu_softc *,
+ struct acpidmar_drhd *);
+int iommu_enable_translation(struct iommu_softc *, int);
+void iommu_enable_qi(struct iommu_softc *, int);
+void iommu_flush_cache(struct iommu_softc *, void *, size_t);
+void *iommu_alloc_page(struct iommu_softc *, paddr_t *);
+
+void iommu_flush_ctx(struct iommu_softc *, int, int, int, int);
+void iommu_flush_tlb(struct iommu_softc *, int, int);
+
+const char *dmar_bdf(int);
+
+const char *
+dmar_bdf(int sid)
+{
+ static char bdf[32];
+
+ snprintf(bdf, sizeof(bdf), "%.4x:%.2x:%.2x.%x", 0,
+ sid_bus(sid), sid_dev(sid), sid_fun(sid));
+
+ return (bdf);
+}
+
+/* busdma */
+static int dmar_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
+ bus_size_t, int, bus_dmamap_t *);
+static void dmar_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
+static int dmar_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
+ struct proc *, int);
+static int dmar_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t, struct mbuf *,
+ int);
+static int dmar_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t, struct uio *,
int);
+static int dmar_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t,
+ bus_dma_segment_t *, int, bus_size_t, int);
+static void dmar_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
+static void dmar_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
+ bus_size_t, int);
+static int dmar_dmamem_alloc(bus_dma_tag_t, bus_size_t, bus_size_t, bus_size_t,
+ bus_dma_segment_t *, int, int *, int);
+static void dmar_dmamem_free(bus_dma_tag_t, bus_dma_segment_t *, int);
+static int dmar_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *, int, size_t,
+ caddr_t *, int);
+static void dmar_dmamem_unmap(bus_dma_tag_t, caddr_t, size_t);
+static paddr_t dmar_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *, int, off_t,
+ int, int);
+
+int dmar_dbg = 0;
+
+static inline int
+debugme(struct domain *dom)
+{
+ return (0);
+}
+
+#if 0
+void
+dmar_printf(const char *fmt, ...)
+{
+ va_list ap;
+
+ return;
+ va_start(ap,fmt);
+ vprintf(fmt,ap);
+}
+#endif
+
+const char *dom_bdf(struct domain *dom);
+
+/* Map a range of pages 1:1 */
+void
+domain_map_pthru(struct domain *dom, paddr_t start, paddr_t end)
+{
+ while (start < end) {
+ domain_map_page(dom, start, start, PTE_P | PTE_R | PTE_W);
+ start += VTD_PAGE_SIZE;
+ }
+}
+
+/* Map a single paddr to IOMMU paddr */
+void
+domain_map_page(struct domain *dom, vaddr_t va, paddr_t pa, int flags)
+{
+ paddr_t paddr;
+ struct pte_entry *pte, *npte;
+ int lvl, idx;
+
+ /* Only handle 4k pages for now */
+ npte = dom->pte;
+ /* XXX why 12? */
+ for (lvl = dom->iommu->agaw - VTD_STRIDE_SIZE; lvl>= 12;
+ lvl -= VTD_STRIDE_SIZE) {
+ idx = (va>> lvl) & VTD_STRIDE_MASK;
+ pte = &npte[idx];
+ if (lvl == 12) {
+ /* Level 1: Page Table - add physical address */
+ pte->val = pa | flags;
+ iommu_flush_cache(dom->iommu, pte, sizeof(*pte));
+ break;
+ } else if (!pte->val & PTE_P) {
+ /* Level N: Point to lower level table */
+ iommu_alloc_page(dom->iommu, &paddr);
+ pte->val = paddr | PTE_P | PTE_R | PTE_W;
+ iommu_flush_cache(dom->iommu, pte, sizeof(*pte));
+ }
+ npte = (void *)PMAP_DIRECT_MAP((pte->val & ~0xFFF));
+ }
+}
+
+/* Unload mapping */
+void
+domain_unload_map(struct domain *dom, bus_dmamap_t dmam)
+{
+ bus_dma_segment_t *seg;
+ paddr_t base, end, idx;
+ psize_t alen;
+ int i, s;
+
+ if (iommu_bad(dom->iommu)) {
+ printf("unload map no iommu\n");
+ return;
+ }
+
+ acpidmar_intr(dom->iommu);
+ for (i = 0; i < dmam->dm_nsegs; i++) {
+ seg = &dmam->dm_segs[i];
+
+ base = trunc_page(seg->ds_addr);
+ end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE);
+ alen = end - base;
+
+ if (debugme(dom)) {
+ printf(" va:%.16llx len:%x\n",
+ (uint64_t)base, (uint32_t)alen);
+ }
+
+ s = splhigh();
+ extent_free(dom->iovamap, base, alen, EX_NOWAIT);
+ splx(s);
+
+ /* Clear PTE */
+ for (idx = 0; idx < alen; idx+=VTD_PAGE_SIZE)
+ domain_map_page(dom, base + idx, 0, 0);
+ }
+}
+
+/* map.segs[x].ds_addr is modified to IOMMU virtual PA */
+void
+domain_load_map(struct domain *dom, bus_dmamap_t map, int flags, const char
*fn)
+{
+ bus_dma_segment_t *seg;
+ struct iommu_softc *iommu;
+ paddr_t base, end, idx;
+ psize_t alen;
+ u_long res;
+ int i, s;
+
+ iommu = dom->iommu;
+ if (!iommu_enabled(iommu)) {
+ /* Lazy enable translation when required */
+ if (iommu_enable_translation(iommu, 1)) {
+ return;
+ }
+ }
+
+ acpidmar_intr(dom->iommu);
+ if (debugme(dom)) {
+ printf("%s: %s\n", fn, dom_bdf(dom));
+ }
+
+ for (i = 0; i < map->dm_nsegs; i++) {
+ seg = &map->dm_segs[i];
+
+ base = trunc_page(seg->ds_addr);
+ end = roundup(seg->ds_addr + seg->ds_len, VTD_PAGE_SIZE);
+ alen = end - base;
+
+ s = splhigh();
+ extent_alloc(dom->iovamap, alen, VTD_PAGE_SIZE, 0,
+ map->_dm_boundary, EX_NOWAIT, &res);
+ splx(s);
+
+ if (debugme(dom)) {
+ printf(" %.16llx %x => %.16llx\n",
+ (uint64_t)seg->ds_addr, (uint32_t)seg->ds_len,
+ (uint64_t)res);
+ }
+
+ seg->ds_addr = res | (seg->ds_addr & 0xFFF);
+ for (idx = 0; idx < alen; idx += VTD_PAGE_SIZE) {
+ domain_map_page(dom, res + idx, base + idx,
+ PTE_P | PTE_R | PTE_W);
+ }
+ }
+
+ iommu_flush_tlb(dom->iommu, IOTLB_DOMAIN, dom->did);
+}
+
+/* Bus DMA Map functions */
+const char *dom_bdf(struct domain *dom)
+{
+ struct domain_dev *dd;
+ static char mmm[48];
+
+ dd = TAILQ_FIRST(&dom->devices);
+ snprintf(mmm, sizeof(mmm), "%s iommu:%d did:%.4x%s",
+ dmar_bdf(dd->sid), dom->iommu->id, dom->did,
+ dom->did == DID_UNITY ? " [unity]" : "");
+
+ return (mmm);
+}
+
+static int
+dmar_dmamap_create(bus_dma_tag_t tag, bus_size_t size, int nsegments,
+ bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
+{
+ struct domain *dom = tag->_cookie;
+ bus_dmamap_t dmam;
+ int rc, i;
+
+ rc=_bus_dmamap_create(tag,size,nsegments,maxsegsz,boundary,
+ flags,dmamp);
+ if (rc)
+ return (rc);
+
+ dmam = *dmamp;
+ if (debugme(dom)) {
+ printf("dmamap_create: %s\n", dom_bdf(dom));
+ for (i = 0; i<dmam->dm_nsegs; i++) {
+ printf(" create: %.16llx %x\n",
+ (uint64_t)dmam->dm_segs[i].ds_addr,
+ (uint32_t)dmam->dm_segs[i].ds_len);
+ }
+ }
+
+ return (rc);
+}
+
+static void
+dmar_dmamap_destroy(bus_dma_tag_t tag, bus_dmamap_t dmam)
+{
+ struct domain *dom = tag->_cookie;
+ int i;
+
+ if (debugme(dom)) {
+ printf("dmamap_destroy: %s\n", dom_bdf(dom));
+ for (i = 0; i < dmam->dm_nsegs; i++) {
+ printf(" destroy: %.16llx %x\n",
+ (uint64_t)dmam->dm_segs[i].ds_addr,
+ (uint32_t)dmam->dm_segs[i].ds_len);
+ }
+ }
+ _bus_dmamap_destroy(tag,dmam);
+}
+
+static int
+dmar_dmamap_load(bus_dma_tag_t tag, bus_dmamap_t dmam, void *buf,
+ bus_size_t buflen, struct proc *p, int flags)
+{
+ struct domain *dom = tag->_cookie;
+ int rc;
+
+ rc=_bus_dmamap_load(tag,dmam,buf,buflen,p,flags);
+ if (rc)
+ return (rc);
+
+ domain_load_map(dom, dmam, flags, __FUNCTION__);
+
+ return (0);
+}
+
+static int
+dmar_dmamap_load_mbuf(bus_dma_tag_t tag, bus_dmamap_t dmam, struct mbuf *chain,
+ int flags)
+{
+ struct domain *dom = tag->_cookie;
+ int rc;
+
+ rc=_bus_dmamap_load_mbuf(tag,dmam,chain,flags);
+ if (rc)
+ return (rc);
+
+ domain_load_map(dom, dmam, flags, __FUNCTION__);
+
+ return (0);
+}
+
+static int
+dmar_dmamap_load_uio(bus_dma_tag_t tag, bus_dmamap_t dmam, struct uio *uio,
+ int flags)
+{
+ struct domain *dom = tag->_cookie;
+ int rc;
+
+ rc=_bus_dmamap_load_uio(tag,dmam,uio,flags);
+ if (rc)
+ return (rc);
+
+ domain_load_map(dom, dmam, flags, __FUNCTION__);
+
+ return (0);
+}
+
+static int
+dmar_dmamap_load_raw(bus_dma_tag_t tag, bus_dmamap_t dmam,
+ bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags)
+{
+ struct domain *dom = tag->_cookie;
+ int rc;
+
+ rc=_bus_dmamap_load_raw(tag,dmam,segs,nsegs,size,flags);
+ if (rc)
+ return rc;
+
+ domain_load_map(dom, dmam, flags, __FUNCTION__);
+
+ return (0);
+}
+
+static void
+dmar_dmamap_unload(bus_dma_tag_t tag, bus_dmamap_t dmam)
+{
+ struct domain *dom = tag->_cookie;
+
+ if (debugme(dom)) {
+ printf("dmamap_unload: %s\n", dom_bdf(dom));
+ }
+ domain_unload_map(dom, dmam);
+ _bus_dmamap_unload(tag,dmam);
+}
+
+static void
+dmar_dmamap_sync(bus_dma_tag_t tag, bus_dmamap_t dmam, bus_addr_t offset,
+ bus_size_t len, int ops)
+{
+ struct domain *dom = tag->_cookie;
+ int i, flag;
+
+ flag = 0;
+ acpidmar_intr(dom->iommu);
+ if (ops == BUS_DMASYNC_PREREAD) {
+ /* make readable */
+ flag = PTE_P | PTE_R;
+ } else if (ops == BUS_DMASYNC_PREWRITE) {
+ /* make writeable */
+ flag = PTE_P | PTE_W;
+ }
+ if (debugme(dom)) {
+ printf("dmamap_sync: %s\n", dom_bdf(dom));
+ for (i=0; i<dmam->dm_nsegs; i++) {
+ printf(" sync: %.16llx %x\n",
+ (uint64_t)dmam->dm_segs[i].ds_addr,
+ (uint32_t)dmam->dm_segs[i].ds_len);
+ }
+ }
+ _bus_dmamap_sync(tag,dmam,offset,len,ops);
+}
+
+static int
+dmar_dmamem_alloc(bus_dma_tag_t tag, bus_size_t size, bus_size_t alignment,
+ bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
+ int flags)
+{
+ struct domain *dom = tag->_cookie;
+ int rc, i;
+
+ rc = _bus_dmamem_alloc(tag,size,alignment,boundary,segs,nsegs,rsegs,
+ flags);
+ if (rc)
+ return rc;
+
+ if (debugme(dom)) {
+ printf("dmamap_alloc size=%llx nsegs=%d %s\n",
+ (uint64_t)size, nsegs, dom_bdf(dom));
+ for (i = 0; i < *rsegs; i++) {
+ printf(" alloc: %.16llx %x\n",
+ (uint64_t)segs[i].ds_addr,
+ (uint32_t)segs[i].ds_len);
+ }
+ }
+
+ return (rc);
+}
+
+static void
+dmar_dmamem_free(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs)
+{
+ struct domain *dom = tag->_cookie;
+ int i;
+
+ if (debugme(dom)) {
+ printf("dmamap_free: %s\n", dom_bdf(dom));
+ for (i = 0; i < nsegs; i++) {
+ printf(" free: %.16llx %x\n",
+ (uint64_t)segs[i].ds_addr,
+ (uint32_t)segs[i].ds_len);
+ }
+ }
+ _bus_dmamem_free(tag,segs,nsegs);
+}
+
+static int
+dmar_dmamem_map(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
+ size_t size, caddr_t *kvap, int flags)
+{
+ struct domain *dom = tag->_cookie;
+ int i;
+
+ if (debugme(dom)) {
+ printf("dmamap_map: %s\n", dom_bdf(dom));
+ for (i = 0; i < nsegs; i++) {
+ printf(" map: %.16llx %x\n",
+ (uint64_t)segs[i].ds_addr,
+ (uint32_t)segs[i].ds_len);
+ }
+ }
+
+ return (_bus_dmamem_map(tag,segs,nsegs,size,kvap,flags));
+}
+
+static void
+dmar_dmamem_unmap(bus_dma_tag_t tag, caddr_t kva, size_t size)
+{
+ struct domain *dom = tag->_cookie;
+
+ if (debugme(dom)) {
+ printf("dmamap_unmap: %s\n", dom_bdf(dom));
+ }
+ _bus_dmamem_unmap(tag,kva,size);
+}
+
+static paddr_t
+dmar_dmamem_mmap(bus_dma_tag_t tag, bus_dma_segment_t *segs, int nsegs,
+ off_t off, int prot, int flags)
+{
+ struct domain *dom = tag->_cookie;
+ int i;
+
+ if (debugme(dom)) {
+ printf("dmamap_mmap: %s\n", dom_bdf(dom));
+ for (i = 0; i < nsegs; i++) {
+ printf(" mmap: %.16llx %.8x\n",
+ (uint64_t)segs[i].ds_addr,
+ (uint32_t)segs[i].ds_len);
+ }
+ }
+
+ return (_bus_dmamem_mmap(tag,segs,nsegs,off,prot,flags));
+}
+
+/*===================================
+ * IOMMU code
+ *==================================*/
+void *
+iommu_alloc_page(struct iommu_softc *iommu, paddr_t *paddr)
+{
+ void *va;
+
+ *paddr = 0;
+ va = km_alloc(VTD_PAGE_SIZE, &kv_page, &kp_zero, &kd_nowait);
+ if (va == NULL) {
+ panic("can't allocate page\n");
+ }
+ pmap_extract(pmap_kernel(), (vaddr_t)va, paddr);
+
+ return (va);
+}
+
+void
+iommu_flush_cache(struct iommu_softc *iommu, void *addr, size_t size)
+{
+ if (!(iommu->ecap & ECAP_C))
+ pmap_flush_cache((vaddr_t)addr, size);
+}
+
+void
+iommu_flush_tlb(struct iommu_softc *iommu, int mode, int did)
+{
+ int n;
+ uint64_t val;
+
+ if (iommu->gcmd & GCMD_QIE) {
+ struct qi_entry qi;
+
+ /* Use queued invalidation */
+ qi.hi = 0;
+ switch (mode) {
+ case IOTLB_GLOBAL:
+ qi.lo = QI_IOTLB | QI_IOTLB_IG_GLOBAL;
+ break;
+ case IOTLB_DOMAIN:
+ qi.lo = QI_IOTLB | QI_IOTLB_IG_DOMAIN |
+ QI_IOTLB_DID(did);
+ break;
+ case IOTLB_PAGE:
+ qi.lo = QI_IOTLB | QI_IOTLB_IG_PAGE | QI_IOTLB_DID(did);
+ qi.hi = 0;
+ break;
+ }
+ if (iommu->cap & CAP_DRD)
+ qi.lo |= QI_IOTLB_DR;
+ if (iommu->cap & CAP_DWD)
+ qi.lo |= QI_IOTLB_DW;
+ return;
+ }
+
+ val = IOTLB_IVT;
+ switch (mode) {
+ case IOTLB_GLOBAL:
+ val |= IIG_GLOBAL;
+ break;
+ case IOTLB_DOMAIN:
+ val |= IIG_DOMAIN | IOTLB_DID(did);
+ break;
+ case IOTLB_PAGE:
+ val |= IIG_PAGE | IOTLB_DID(did);
+ break;
+ }
+
+ if (iommu->cap & CAP_DRD)
+ val |= IOTLB_DR;
+ if (iommu->cap & CAP_DWD)
+ val |= IOTLB_DW;
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu_writeq(iommu, DMAR_IOTLB_REG(iommu), val);
+ n = 0;
+ do {
+ val = iommu_readq(iommu, DMAR_IOTLB_REG(iommu));
+ } while (n++ < 5 && val & IOTLB_IVT);
+
+ mtx_leave(&iommu->reg_lock);
+
+#ifdef DEBUG
+ {
+ static int rg;
+ int a, r;
+
+ if (!rg) {
+ a = (val>> IOTLB_IAIG_SHIFT) & IOTLB_IAIG_MASK;
+ r = (val>> IOTLB_IIRG_SHIFT) & IOTLB_IIRG_MASK;
+ if (a != r) {
+ printf("TLB Requested:%d Actual:%d\n", r, a);
+ rg = 1;
+ }
+ }
+ }
+#endif
+}
+
+void
+iommu_flush_ctx(struct iommu_softc *iommu, int mode, int did, int sid, int fm)
+{
+ uint64_t val;
+ int n;
+
+ if (iommu->gcmd & GCMD_QIE) {
+ struct qi_entry qi;
+
+ /* Use queued invalidation */
+ qi.hi = 0;
+ switch (mode) {
+ case CTX_GLOBAL:
+ qi.lo = QI_CTX | QI_CTX_IG_GLOBAL;
+ break;
+ case CTX_DOMAIN:
+ qi.lo = QI_CTX | QI_CTX_IG_DOMAIN | QI_CTX_DID(did);
+ break;
+ case CTX_DEVICE:
+ qi.lo = QI_CTX | QI_CTX_IG_DEVICE | QI_CTX_DID(did) |
+ QI_CTX_SID(sid) | QI_CTX_FM(fm);
+ break;
+ }
+ return;
+ }
+
+ val = CCMD_ICC;
+ switch (mode) {
+ case CTX_GLOBAL:
+ val |= CIG_GLOBAL;
+ break;
+ case CTX_DOMAIN:
+ val |= CIG_DOMAIN | CCMD_DID(did);
+ break;
+ case CTX_DEVICE:
+ val |= CIG_DEVICE | CCMD_DID(did) |
+ CCMD_SID(sid) | CCMD_FM(fm);
+ break;
+ }
+
+ mtx_enter(&iommu->reg_lock);
+
+ n = 0;
+ iommu_writeq(iommu, DMAR_CCMD_REG, val);
+ do {
+ val = iommu_readq(iommu, DMAR_CCMD_REG);
+ } while (n++ < 5 && val & CCMD_ICC);
+
+ mtx_leave(&iommu->reg_lock);
+
+#ifdef DEBUG
+ {
+ static int rg;
+ int a, r;
+
+ if (!rg) {
+ a = (val>> CCMD_CAIG_SHIFT) & CCMD_CAIG_MASK;
+ r = (val>> CCMD_CIRG_SHIFT) & CCMD_CIRG_MASK;
+ if (a != r) {
+ printf("CTX Requested:%d Actual:%d\n", r, a);
+ rg = 1;
+ }
+ }
+ }
+#endif
+}
+
+void
+iommu_enable_qi(struct iommu_softc *iommu, int enable)
+{
+ int n = 0;
+ int sts;
+
+ if (!(iommu->ecap & ECAP_QI))
+ return;
+
+ if (enable) {
+ iommu->gcmd |= GCMD_QIE;
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu_writel(iommu, DMAR_GCMD_REG, iommu->gcmd);
+ do {
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ } while (n++ < 5 && !(sts & GSTS_QIES));
+
+ mtx_leave(&iommu->reg_lock);
+
+ printf("set.qie: %d\n", n);
+ } else {
+ iommu->gcmd &= ~GCMD_QIE;
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu_writel(iommu, DMAR_GCMD_REG, iommu->gcmd);
+ do {
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ } while (n++ < 5 && sts & GSTS_QIES);
+
+ mtx_leave(&iommu->reg_lock);
+
+ printf("clr.qie: %d\n", n);
+ }
+}
+
+int
+iommu_enable_translation(struct iommu_softc *iommu, int enable)
+{
+ uint32_t sts;
+ uint64_t reg;
+ int n = 0;
+
+ reg = 0;
+ if (enable) {
+ printf("enable iommu %d\n", iommu->id);
+ iommu_showcfg(iommu);
+
+ iommu->gcmd |= GCMD_TE;
+
+ /* Enable translation */
+ printf(" pre tes: ");
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu_writel(iommu, DMAR_GCMD_REG, iommu->gcmd);
+ printf("xxx");
+ do {
+ printf("yyy");
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ } while (n++ < 5 && !(sts & GSTS_TES));
+
+ mtx_leave(&iommu->reg_lock);
+
+ printf(" set.tes: %d\n", n);
+
+ if (n>= 5) {
+ printf("error.. unable to initialize iommu %d\n",
+ iommu->id);
+ iommu->flags |= IOMMU_FLAGS_BAD;
+
+ /* Disable IOMMU */
+ iommu->gcmd &= ~GCMD_TE;
+ mtx_enter(&iommu->reg_lock);
+ iommu_writel(iommu, DMAR_GCMD_REG, iommu->gcmd);
+ mtx_leave(&iommu->reg_lock);
+
+ return (1);
+ }
+
+ iommu_flush_ctx(iommu, CTX_GLOBAL, 0, 0, 0);
+ iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
+ } else {
+ iommu->gcmd &= ~GCMD_TE;
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu_writel(iommu, DMAR_GCMD_REG, iommu->gcmd);
+ do {
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ } while (n++ < 5 && sts & GSTS_TES);
+
+ mtx_leave(&iommu->reg_lock);
+
+ printf(" clr.tes: %d\n", n);
+ }
+
+ return (0);
+}
+
+int
+iommu_init(struct acpidmar_softc *sc, struct iommu_softc *iommu,
+ struct acpidmar_drhd *dh)
+{
+ static int niommu;
+ int len = VTD_PAGE_SIZE;
+ int i, gaw;
+ uint32_t sts;
+ paddr_t paddr;
+
+ if (_bus_space_map(sc->sc_memt, dh->address, len, 0, &iommu->ioh) != 0)
{
+ return (-1);
+ }
+
+ TAILQ_INIT(&iommu->domains);
+ iommu->id = ++niommu;
+ iommu->flags = dh->flags;
+ iommu->segment = dh->segment;
+ iommu->iot = sc->sc_memt;
+
+ iommu->cap = iommu_readq(iommu, DMAR_CAP_REG);
+ iommu->ecap = iommu_readq(iommu, DMAR_ECAP_REG);
+ iommu->ndoms = cap_nd(iommu->cap);
+
+ mtx_init(&iommu->reg_lock, IPL_HIGH);
+
+ /* Clear Interrupt Masking */
+ iommu->intr = acpidmar_intr_establish(iommu, IPL_BIO, acpidmar_intr,
+ iommu, "dmarintr");
+
+ /* Allocate root pointer */
+ iommu->root = iommu_alloc_page(iommu, &paddr);
+#ifdef DEBUG
+ printf("Allocated root pointer: pa:%.16llx va:%p\n",
+ (uint64_t)paddr, iommu->root);
+#endif
+ iommu->rtaddr = paddr;
+ i = 0;
+ iommu_writeq(iommu, DMAR_RTADDR_REG, paddr);
+ iommu_writel(iommu, DMAR_GCMD_REG, iommu->gcmd | GCMD_SRTP);
+ do {
+ /* XXX shoule we sleep here? */
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ } while (++i < 5 && !(sts & GSTS_RTPS));
+ dprintf("set.rtp = %d\n", i);
+
+ /* Disable interrupts */
+ sts = iommu_readl(iommu, DMAR_FECTL_REG);
+ iommu_writel(iommu, DMAR_FECTL_REG, sts | FECTL_IM);
+#if 0
+ if (iommu->ecap & ECAP_QI) {
+ /* Queued Invalidation support */
+ iommu->qi = iommu_alloc_page(iommu, &iommu->qip);
+ iommu_writeq(iommu, DMAR_IQT_REG, 0);
+ iommu_writeq(iommu, DMAR_IQA_REG, iommu->qip | IQA_QS_256);
+ }
+ if (iommu->ecap & ECAP_IR) {
+ /* Interrupt remapping support */
+ iommu_writeq(iommu, DMAR_IRTA_REG, 0);
+ }
+#endif
+
+#if 0
+ printf(" pt:%d c:%d dt:%d zlr:%d dwd:%d drd:%d cm:%d domains:%d ",
+ !!(iommu->ecap & ECAP_PT), // passthrough
+ !!(iommu->ecap & ECAP_C), // coherent
+ !!(iommu->ecap & ECAP_DT), // device iotlb
+ !!(iommu->cap & CAP_ZLR), // zero-length read
+ !!(iommu->cap & CAP_DWD), // write draining
+ !!(iommu->cap & CAP_DRD), // read draining
+ !!(iommu->cap & CAP_CM), // caching mode
+ iommu->ndoms);
+#endif
+ gaw = -1;
+ iommu->mgaw = cap_mgaw(iommu->cap);
+ for (i = 0; i < 5; i++) {
+ if (cap_sagaw(iommu->cap) & (1L << i)) {
+ gaw = (i * VTD_STRIDE_SIZE) + 30;
+ iommu->agaw = gaw;
+ }
+ }
+
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ if (sts & GSTS_TES)
+ iommu->gcmd |= GCMD_TE;
+ if (sts & GSTS_QIES)
+ iommu->gcmd |= GCMD_QIE;
+ if (sts & GSTS_IRES)
+ iommu->gcmd |= GCMD_IRE;
+
+ return (0);
+}
+
+const char *dmar_rn(int reg);
+
+const char *
+dmar_rn(int reg)
+{
+ switch (reg) {
+ case DMAR_VER_REG: return "ver";
+ case DMAR_CAP_REG: return "cap";
+ case DMAR_ECAP_REG: return "ecap";
+ case DMAR_GSTS_REG: return "gsts";
+ case DMAR_GCMD_REG: return "gcmd";
+ case DMAR_FSTS_REG: return "fsts";
+ case DMAR_FECTL_REG: return "fectl";
+ case DMAR_RTADDR_REG: return "rtaddr";
+ case DMAR_FEDATA_REG: return "fedata";
+ case DMAR_FEADDR_REG: return "feaddr";
+ case DMAR_FEUADDR_REG: return "feuaddr";
+ case DMAR_PMEN_REG: return "pmen";
+ case DMAR_IEDATA_REG: return "iedata";
+ case DMAR_IEADDR_REG: return "ieaddr";
+ case DMAR_IEUADDR_REG: return "ieuaddr";
+ case DMAR_IRTA_REG: return "irta";
+ case DMAR_CCMD_REG: return "ccmd";
+ case DMAR_IQH_REG: return "iqh";
+ case DMAR_IQT_REG: return "iqt";
+ case DMAR_IQA_REG: return "iqa";
+ }
+ return "unknown";
+}
+
+/* Read/Write IOMMU register */
+uint32_t
+iommu_readl(struct iommu_softc *iommu, int reg)
+{
+ uint32_t v;
+
+ v = bus_space_read_4(iommu->iot, iommu->ioh, reg);
+ if (reg < 00) {
+ printf("iommu%d: read %x %.8x [%s]\n",
+ iommu->id, reg, v, dmar_rn(reg));
+ }
+
+ return (v);
+}
+
+uint64_t
+iommu_readq(struct iommu_softc *iommu, int reg)
+{
+ uint64_t v;
+
+ v = bus_space_read_8(iommu->iot, iommu->ioh, reg);
+ if (reg < 00) {
+ printf("iommu%d: read %x %.16llx [%s]\n",
+ iommu->id, reg, v, dmar_rn(reg));
+ }
+
+ return (v);
+}
+
+void
+iommu_writel(struct iommu_softc *iommu, int reg, uint32_t v)
+{
+ if (reg < 00) {
+ printf("iommu%d: write %x %.8x [%s]\n",
+ iommu->id, reg, v, dmar_rn(reg));
+ }
+ bus_space_write_4(iommu->iot, iommu->ioh, reg, v);
+}
+
+void
+iommu_writeq(struct iommu_softc *iommu, int reg, uint64_t v)
+{
+ if (reg < 00) {
+ printf("iommu%d: write %x %.16llx [%s]\n",
+ iommu->id, reg, v, dmar_rn(reg));
+ }
+ bus_space_write_8(iommu->iot, iommu->ioh, reg, v);
+}
+
+/* Check if a device is within a device scope */
+int
+acpidmar_match_devscope(struct devlist_head *devlist, pci_chipset_tag_t pc,
+ int sid)
+{
+ struct dmar_devlist *ds;
+ int sub, sec, i;
+ int bus, dev, fun, sbus;
+ pcireg_t reg;
+ pcitag_t tag;
+
+ sbus = sid_bus(sid);
+ TAILQ_FOREACH(ds, devlist, link) {
+ bus = ds->bus;
+ dev = ds->dp[0].device;
+ fun = ds->dp[0].function;
+ /* Walk PCI bridges in path */
+ for (i = 1; i < ds->ndp; i++) {
+ tag = pci_make_tag(pc, bus, dev, fun);
+ reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO);
+ bus = PPB_BUSINFO_SECONDARY(reg);
+ dev = ds->dp[i].device;
+ fun = ds->dp[i].function;
+ }
+
+ /* Check for device exact match */
+ if (sid == mksid(bus, dev, fun)) {
+ return DMAR_ENDPOINT;
+ }
+
+ /* Check for device subtree match */
+ if (ds->type == DMAR_BRIDGE) {
+ tag = pci_make_tag(pc, bus, dev, fun);
+ reg = pci_conf_read(pc, tag, PPB_REG_BUSINFO);
+ sec = PPB_BUSINFO_SECONDARY(reg);
+ sub = PPB_BUSINFO_SUBORDINATE(reg);
+ if (sec <= sbus && sbus <= sub) {
+ return DMAR_BRIDGE;
+ }
+ }
+ }
+
+ return (0);
+}
+
+struct domain *
+domain_create(struct iommu_softc *iommu, int did)
+{
+ struct domain *dom;
+
+ dom = malloc(sizeof(*dom), M_DEVBUF, M_ZERO | M_WAITOK);
+ dom->did = did;
+ dom->iommu = iommu;
+ dom->pte = iommu_alloc_page(iommu, &dom->ptep);
+ TAILQ_INIT(&dom->devices);
+
+ /* Setup DMA */
+ dom->dmat._cookie = dom;
+ dom->dmat._dmamap_create = dmar_dmamap_create;
+ dom->dmat._dmamap_destroy = dmar_dmamap_destroy;
+ dom->dmat._dmamap_load = dmar_dmamap_load;
+ dom->dmat._dmamap_load_mbuf = dmar_dmamap_load_mbuf;
+ dom->dmat._dmamap_load_uio = dmar_dmamap_load_uio;
+ dom->dmat._dmamap_load_raw = dmar_dmamap_load_raw;
+ dom->dmat._dmamap_unload = dmar_dmamap_unload;
+ dom->dmat._dmamap_sync = dmar_dmamap_sync;
+ dom->dmat._dmamem_alloc = dmar_dmamem_alloc;
+ dom->dmat._dmamem_free = dmar_dmamem_free;
+ dom->dmat._dmamem_map = dmar_dmamem_map;
+ dom->dmat._dmamem_unmap = dmar_dmamem_unmap;
+ dom->dmat._dmamem_mmap = dmar_dmamem_mmap;
+
+ snprintf(dom->exname, sizeof(dom->exname), "did:%x.%.4x",
+ iommu->id, dom->did);
+
+ /* Setup IOMMU address map */
+ dom->iovamap = extent_create(dom->exname, 1024*1024*16,
+ (1LL << iommu->agaw)-1,
+ M_DEVBUF, NULL, 0,
+ EX_WAITOK|EX_NOCOALESCE);
+
+ /* Zero out Interrupt region */
+ extent_alloc_region(dom->iovamap, 0xFEE00000L, 0x100000,
+ EX_WAITOK);
+
+ TAILQ_INSERT_TAIL(&iommu->domains, dom, link);
+
+ return dom;
+}
+
+/* Lookup domain by segment & source id (bus.device.function) */
+struct domain *
+domain_lookup(struct acpidmar_softc *sc, int segment, int sid)
+{
+ struct iommu_softc *iommu;
+ struct domain *dom;
+ struct domain_dev *ddev;
+ int rc;
+
+ if (sc == NULL) {
+ return NULL;
+ }
+
+ /* Lookup IOMMU for this device */
+ TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
+ if (iommu->segment != segment)
+ continue;
+ /* Check for devscope match or catchall iommu */
+ rc=acpidmar_match_devscope(&iommu->devices, sc->sc_pc, sid);
+ if (rc != 0 || iommu->flags) {
+ break;
+ }
+ }
+ if (!iommu) {
+ printf("%s: no iommu found\n", dmar_bdf(sid));
+ return NULL;
+ }
+ /* Search domain devices */
+ TAILQ_FOREACH(dom, &iommu->domains, link) {
+ TAILQ_FOREACH(ddev, &dom->devices, link) {
+ /* XXX: match all functions? */
+ if (ddev->sid == sid) {
+ return dom;
+ }
+ }
+ }
+ if (iommu->ndoms <= 2) {
+ /* Running out of domains.. create catchall domain */
+ if (!iommu->unity) {
+ iommu->unity = domain_create(iommu, 1);
+ }
+ dom = iommu->unity;
+ } else {
+ dom = domain_create(iommu, --iommu->ndoms);
+ }
+ if (!dom) {
+ printf("no domain here\n");
+ return NULL;
+ }
+
+ /* Add device to domain */
+ ddev = malloc(sizeof(*ddev), M_DEVBUF, M_ZERO | M_WAITOK);
+ ddev->sid = sid;
+ TAILQ_INSERT_TAIL(&dom->devices, ddev, link);
+
+ printf("pci bus %d device %d function %d is iommu %d domain %.4x\n",
+ sid_bus(sid), sid_dev(sid), sid_fun(sid),
+ iommu->id, dom->did);
+
+ return dom;
+}
+
+struct domain *
+acpidmar_pci_attach(struct acpidmar_softc *sc, int segment, int sid)
+{
+ static struct domain *dom;
+ struct iommu_softc *iommu;
+ struct context_entry *ctx;
+ paddr_t paddr;
+ int bus, devfn;
+ int tt, lvl;
+
+ bus = sid_bus(sid);
+ devfn = sid_devfn(sid);
+
+ dom = domain_lookup(sc, segment, sid);
+ if (!dom) {
+ printf("no domain: %s\n", dmar_bdf(sid));
+ return NULL;
+ }
+ iommu = dom->iommu;
+
+ /* Create Bus mapping */
+ if (!root_entry_is_valid(&iommu->root[bus])) {
+ iommu->ctx[bus] = iommu_alloc_page(iommu, &paddr);
+ iommu->root[bus].lo = paddr | ROOT_P;
+ iommu_flush_cache(iommu, &iommu->root[bus],
+ sizeof(struct root_entry));
+ dprintf("iommu%d: Allocate context for bus: %.2x pa:%.16llx
va:%p\n",
+ iommu->id, bus, (uint64_t)paddr,
+ iommu->ctx[bus]);
+ }
+
+ /* Create DevFn mapping */
+ ctx = iommu->ctx[bus] + devfn;
+ if (!context_entry_is_valid(ctx)) {
+ tt = CTX_T_MULTI;
+ lvl = (iommu->agaw - 30)/VTD_STRIDE_SIZE;
+
+ dprintf("pci: %s\n", dom_bdf(dom));
+
+ /* Initialize context */
+ context_set_slpte(ctx, dom->ptep);
+ context_set_translation_type(ctx, tt);
+ context_set_domain_id(ctx, dom->did);
+ context_set_address_width(ctx, lvl);
+ context_set_present(ctx);
+
+ /* Flush it */
+ iommu_flush_cache(iommu, ctx, sizeof(struct context_entry));
+ iommu_flush_ctx(iommu, CTX_DEVICE, dom->did, sid, 0);
+ iommu_flush_tlb(iommu, IOTLB_GLOBAL, 0);
+ dprintf("iommu%d: %s set context ptep:%.16llx lvl:%d did:%.4x
tt:%d\n",
+ iommu->id, dmar_bdf(sid), (uint64_t)dom->ptep, lvl,
+ dom->did, tt);
+ }
+ return dom;
+}
+
+void
+acpidmar_pci_hook(pci_chipset_tag_t pc, struct pci_attach_args *pa)
+{
+ int bus, dev, fun;
+ struct domain *dom;
+ pcireg_t reg;
+
+ if (!acpidmar_sc) {
+ /* No DMAR, ignore */
+ return;
+ }
+ pci_decompose_tag(pc, pa->pa_tag, &bus, &dev, &fun);
+ reg = pci_conf_read(pc, pa->pa_tag, PCI_CLASS_REG);
+ if (PCI_CLASS(reg) == PCI_CLASS_DISPLAY &&
+ PCI_SUBCLASS(reg) == PCI_SUBCLASS_DISPLAY_VGA) {
+ printf("dmar: %.4x:%.2x:%.2x.%x is VGA, ignoring\n",
+ pa->pa_domain, bus, dev, fun);
+ return;
+ }
+
+ /* Add device to domain */
+ dom = acpidmar_pci_attach(acpidmar_sc, pa->pa_domain,
+ mksid(bus, dev, fun));
+
+ if (dom != NULL && !iommu_bad(dom->iommu)) {
+ /* Change DMA tag */
+ pa->pa_dmat = &dom->dmat;
+ }
+}
+
+/* Create list of device scope entries from ACPI table */
+void
+acpidmar_parse_devscope(union acpidmar_entry *de, int off, int segment,
+ struct devlist_head *devlist)
+{
+ struct acpidmar_devscope *ds;
+ struct dmar_devlist *d;
+ int dplen, i;
+
+ TAILQ_INIT(devlist);
+ while (off < de->length) {
+ ds = (struct acpidmar_devscope *)((unsigned char *)de + off);
+ off += ds->length;
+
+ /* We only care about bridges and endpoints */
+ if (ds->type != DMAR_ENDPOINT && ds->type != DMAR_BRIDGE)
+ continue;
+
+ dplen = ds->length - 6; /* XXX make this a #define */
+ d = malloc(sizeof(*d) + dplen, M_DEVBUF, M_ZERO | M_WAITOK);
+ d->bus = ds->bus;
+ d->type = ds->type;
+ d->ndp = dplen / 2;
+ d->dp = (void *)&d[1];
+ memcpy(d->dp, &ds[1], dplen);
+ TAILQ_INSERT_TAIL(devlist, d, link);
+
+ printf(" devscope: %8s %.4x:%.2x.%.2x.%x {",
+ ds->type == DMAR_BRIDGE ? "bridge" : "endpoint",
+ segment, ds->bus,
+ d->dp[0].device,
+ d->dp[0].function);
+
+ for (i = 1; i < d->ndp; i++) {
+ printf(" %2x.%x ",
+ d->dp[i].device,
+ d->dp[i].function);
+ }
+ printf("}\n");
+ }
+}
+
+/* DMA Remapping Hardware Unit */
+void
+acpidmar_drhd(struct acpidmar_softc *sc, union acpidmar_entry *de)
+{
+ struct iommu_softc *iommu;
+
+ printf("DRHD: segment:%.4x base:%.16llx flags:%.2x\n",
+ de->drhd.segment,
+ de->drhd.address,
+ de->drhd.flags);
+ iommu = malloc(sizeof(*iommu), M_DEVBUF, M_ZERO | M_WAITOK);
+ acpidmar_parse_devscope(de, sizeof(de->drhd), de->drhd.segment,
+ &iommu->devices);
+ iommu_init(sc, iommu, &de->drhd);
+
+ if (de->drhd.flags) {
+ /* Catchall IOMMU goes at end of list */
+ TAILQ_INSERT_TAIL(&sc->sc_drhds, iommu, link);
+ } else {
+ TAILQ_INSERT_HEAD(&sc->sc_drhds, iommu, link);
+ }
+}
+
+/* Reserved Memory Region Reporting */
+void
+acpidmar_rmrr(struct acpidmar_softc *sc, union acpidmar_entry *de)
+{
+ struct rmrr_softc *rmrr;
+ bios_memmap_t *im;
+
+ printf("RMRR: segment:%.4x range:%.16llx-%.16llx\n",
+ de->rmrr.segment, de->rmrr.base, de->rmrr.limit);
+ if (de->rmrr.limit <= de->rmrr.base) {
+ printf(" buggy BIOS\n");
+ return;
+ }
+
+ rmrr = malloc(sizeof(*rmrr), M_DEVBUF, M_ZERO | M_WAITOK);
+ rmrr->start = trunc_page(de->rmrr.base);
+ rmrr->end = round_page(de->rmrr.limit);
+ rmrr->segment = de->rmrr.segment;
+ acpidmar_parse_devscope(de, sizeof(de->rmrr), de->rmrr.segment,
+ &rmrr->devices);
+
+ for (im = bios_memmap; im->type != BIOS_MAP_END; im++) {
+ if (im->type != BIOS_MAP_RES)
+ continue;
+ if (rmrr->start>= im->addr &&
+ rmrr->end <= im->addr+im->size) {
+ /* Bah.. some buggy BIOS stomp outside RMRR */
+ printf(" ** inside E820 Reserved %.16llx %.16llx\n",
+ im->addr, im->addr+im->size);
+ rmrr->start = trunc_page(im->addr);
+ rmrr->end = round_page(im->addr + im->size);
+ }
+ }
+ TAILQ_INSERT_TAIL(&sc->sc_rmrrs, rmrr, link);
+}
+
+/* Root Port ATS Reporting */
+void
+acpidmar_atsr(struct acpidmar_softc *sc, union acpidmar_entry *de)
+{
+ struct atsr_softc *atsr;
+
+ printf("ATSR: segment:%.4x flags:%x\n",
+ de->atsr.segment,
+ de->atsr.flags);
+
+ atsr = malloc(sizeof(*atsr), M_DEVBUF, M_ZERO | M_WAITOK);
+ atsr->flags = de->atsr.flags;
+ atsr->segment = de->atsr.segment;
+ acpidmar_parse_devscope(de, sizeof(de->atsr), de->atsr.segment,
+ &atsr->devices);
+
+ TAILQ_INSERT_TAIL(&sc->sc_atsrs, atsr, link);
+}
+
+void
+acpidmar_init(struct acpidmar_softc *sc, struct acpi_dmar *dmar)
+{
+ struct rmrr_softc *rmrr;
+ struct iommu_softc *iommu;
+ struct domain *dom;
+ struct dmar_devlist *dl;
+ union acpidmar_entry *de;
+ int off, sid, rc;
+
+ printf(": hardware width: %d, intr_remap:%d x2apic_opt_out:%d\n",
+ dmar->haw+1,
+ !!(dmar->flags & 0x1),
+ !!(dmar->flags & 0x2));
+ sc->sc_haw = dmar->haw+1;
+ sc->sc_flags = dmar->flags;
+
+ TAILQ_INIT(&sc->sc_drhds);
+ TAILQ_INIT(&sc->sc_rmrrs);
+ TAILQ_INIT(&sc->sc_atsrs);
+
+ off = sizeof(*dmar);
+ while (off < dmar->hdr.length) {
+ de = (union acpidmar_entry *)((unsigned char *)dmar + off);
+ switch (de->type) {
+ case DMAR_DRHD:
+ acpidmar_drhd(sc, de);
+ break;
+ case DMAR_RMRR:
+ acpidmar_rmrr(sc, de);
+ break;
+ case DMAR_ATSR:
+ acpidmar_atsr(sc, de);
+ break;
+ default:
+ printf("DMAR: unknown %x\n", de->type);
+ break;
+ }
+ off += de->length;
+ }
+
+ /* Map passthrough pages for RMRR */
+ TAILQ_FOREACH(rmrr, &sc->sc_rmrrs, link) {
+ TAILQ_FOREACH(dl, &rmrr->devices, link) {
+ sid = mksid(dl->bus, dl->dp[0].device,
+ dl->dp[0].function);
+ dom = acpidmar_pci_attach(sc, rmrr->segment, sid);
+ if (dom != NULL) {
+ printf("%s map ident: %.16llx %.16llx\n",
+ dom_bdf(dom), rmrr->start, rmrr->end);
+ domain_map_pthru(dom, rmrr->start, rmrr->end);
+ rc = extent_alloc_region(dom->iovamap,
+ rmrr->start, rmrr->end, EX_WAITOK);
+ }
+ }
+ }
+ TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
+ TAILQ_FOREACH(dl, &iommu->devices, link) {
+ sid = mksid(dl->bus, dl->dp[0].device,
+ dl->dp[0].function);
+ acpidmar_pci_attach(sc, iommu->segment, sid);
+ }
+ }
+ printf("========\n");
+}
+
+int
+acpidmar_activate(struct device *self, int act)
+{
+ struct acpidmar_softc *sc = (struct acpidmar_softc *)self;
+ struct iommu_softc *iommu;
+ int i, sts;
+
+ printf("called acpidmar_activate %d %p\n", act, sc);
+
+ if (sc == NULL) {
+ return (0);
+ }
+
+ switch (act) {
+ case DVACT_RESUME:
+ TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
+ i = 0;
+ iommu_writeq(iommu, DMAR_RTADDR_REG, iommu->rtaddr);
+ iommu_writel(iommu, DMAR_GCMD_REG, iommu->gcmd |
GCMD_SRTP);
+ do {
+ /* XXX shoule we sleep here? */
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ } while (++i < 5 && !(sts & GSTS_RTPS));
+ dprintf("set.rtp = %d\n", i);
+
+ iommu_writel(iommu, DMAR_FEDATA_REG, iommu->fedata);
+ iommu_writel(iommu, DMAR_FEADDR_REG, iommu->feaddr);
+ iommu_writel(iommu, DMAR_FEUADDR_REG,
+ iommu->feaddr>> 32);
+ if ((iommu->flags &
(IOMMU_FLAGS_BAD|IOMMU_FLAGS_SUSPEND)) ==
+ IOMMU_FLAGS_SUSPEND) {
+ printf("enable wakeup translation\n");
+ iommu_enable_translation(iommu, 1);
+ }
+ iommu_showcfg(iommu);
+ }
+ break;
+ case DVACT_SUSPEND:
+ TAILQ_FOREACH(iommu, &sc->sc_drhds, link) {
+ printf("iommu suspend\n");
+ iommu->flags |= IOMMU_FLAGS_SUSPEND;
+ iommu_enable_translation(iommu, 0);
+ }
+ break;
+ }
+ return (0);
+}
+
+int
+acpidmar_match(struct device *parent, void *match, void *aux)
+{
+ struct acpi_attach_args *aaa = aux;
+ struct acpi_table_header *hdr;
+
+ /* If we do not have a table, it is not us */
+ if (aaa->aaa_table == NULL)
+ return (0);
+
+ /* If it is an DMAR table, we can attach */
+ hdr = (struct acpi_table_header *)aaa->aaa_table;
+ if (memcmp(hdr->signature, DMAR_SIG, sizeof(DMAR_SIG) - 1) != 0)
+ return (0);
+
+ return (1);
+}
+
+void
+acpidmar_attach(struct device *parent, struct device *self, void *aux)
+{
+ struct acpidmar_softc *sc = (void *)self;
+ struct acpi_attach_args *aaa = aux;
+ struct acpi_dmar *dmar = (struct acpi_dmar *)aaa->aaa_table;
+
+ acpidmar_sc = sc;
+ sc->sc_memt = aaa->aaa_memt;
+ acpidmar_init(sc, dmar);
+}
+
+/* Interrupt shiz */
+void acpidmar_msi_hwmask(struct pic *, int);
+void acpidmar_msi_hwunmask(struct pic *, int);
+void acpidmar_msi_addroute(struct pic *, struct cpu_info *, int, int, int);
+void acpidmar_msi_delroute(struct pic *, struct cpu_info *, int, int, int);
+
+void
+acpidmar_msi_hwmask(struct pic *pic, int pin)
+{
+ struct iommu_pic *ip = (void *)pic;
+ struct iommu_softc *iommu = ip->iommu;
+
+ printf("msi_hwmask\n");
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu_writel(iommu, DMAR_FECTL_REG, FECTL_IM);
+ iommu_readl(iommu, DMAR_FECTL_REG);
+
+ mtx_leave(&iommu->reg_lock);
+}
+
+void
+acpidmar_msi_hwunmask(struct pic *pic, int pin)
+{
+ struct iommu_pic *ip = (void *)pic;
+ struct iommu_softc *iommu = ip->iommu;
+
+ printf("msi_hwunmask\n");
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu_writel(iommu, DMAR_FECTL_REG, 0);
+ iommu_readl(iommu, DMAR_FECTL_REG);
+
+ mtx_leave(&iommu->reg_lock);
+}
+
+void
+acpidmar_msi_addroute(struct pic *pic, struct cpu_info *ci, int pin, int vec,
+ int type)
+{
+ struct iommu_pic *ip = (void *)pic;
+ struct iommu_softc *iommu = ip->iommu;
+ uint32_t sts;
+
+ mtx_enter(&iommu->reg_lock);
+
+ iommu->fedata = vec;
+ iommu->feaddr = 0xfee00000L | (ci->ci_apicid << 12);
+ iommu_writel(iommu, DMAR_FEDATA_REG, vec);
+ iommu_writel(iommu, DMAR_FEADDR_REG, iommu->feaddr);
+ iommu_writel(iommu, DMAR_FEUADDR_REG, iommu->feaddr>> 32);
+ sts = iommu_readl(iommu, DMAR_FECTL_REG);
+ iommu_writel(iommu, DMAR_FECTL_REG, sts * ~FECTL_IM);
+
+ mtx_leave(&iommu->reg_lock);
+}
+
+void
+acpidmar_msi_delroute(struct pic *pic, struct cpu_info *ci, int pin, int vec,
+ int type)
+{
+ printf("msi_delroute\n");
+}
+
+void *
+acpidmar_intr_establish(void *ctx, int level, int (*func)(void *),
+ void *arg, const char *what)
+{
+ struct iommu_softc *iommu = ctx;
+ struct pic *pic;
+
+ pic = &iommu->pic.pic;
+ iommu->pic.iommu = iommu;
+
+ strlcpy(pic->pic_dev.dv_xname, "dmarpic",
+ sizeof(pic->pic_dev.dv_xname));
+ pic->pic_type = PIC_MSI;
+ pic->pic_hwmask = acpidmar_msi_hwmask;
+ pic->pic_hwunmask = acpidmar_msi_hwunmask;
+ pic->pic_addroute = acpidmar_msi_addroute;
+ pic->pic_delroute = acpidmar_msi_delroute;
+ pic->pic_edge_stubs = ioapic_edge_stubs;
+#ifdef MULTIPROCESSOR
+ mtx_init(&pic->pic_mutex, IPL_HIGH);
+#endif
+
+ return intr_establish(-1, pic, 0, IST_PULSE, level, func, arg, what);
+}
+
+int
+acpidmar_intr(void *ctx)
+{
+ struct iommu_softc *iommu = ctx;
+ struct fault_entry fe;
+ static struct fault_entry ofe;
+ int fro, nfr, fri, i;
+ uint32_t sts;
+
+ mtx_enter(&iommu->reg_lock);
+ sts = iommu_readl(iommu, DMAR_FSTS_REG);
+
+ if (!(sts & FSTS_PPF)) {
+ mtx_leave(&iommu->reg_lock);
+ return (1);
+ }
+
+ nfr = cap_nfr(iommu->cap);
+ fro = cap_fro(iommu->cap);
+ fri = (sts>> FSTS_FRI_SHIFT) & FSTS_FRI_MASK;
+ for (i = 0; i < nfr; i++) {
+ fe.hi = iommu_readq(iommu, fro + (fri*16) + 8);
+ if (!(fe.hi & FRCD_HI_F))
+ break;
+
+ fe.lo = iommu_readq(iommu, fro + (fri*16));
+ if (ofe.hi != fe.hi || ofe.lo != fe.lo) {
+ iommu_showfault(iommu, fri, &fe);
+ ofe.hi = fe.hi;
+ ofe.lo = fe.lo;
+ }
+ fri = (fri + 1) % nfr;
+ }
+
+ iommu_writel(iommu, DMAR_FSTS_REG, FSTS_PFO | FSTS_PPF);
+ mtx_leave(&iommu->reg_lock);
+
+ return (1);
+}
+
+enum {
+ VTD_FAULT_ROOT_P = 0x1, /* P field in root entry is 0 */
+ VTD_FAULT_CTX_P = 0x2, /* P field in context entry is 0 */
+ VTD_FAULT_CTX_INVAL = 0x3, /* context AW/TT/SLPPTR invalid */
+ VTD_FAULT_LIMIT = 0x4, /* Address is outside of MGAW */
+ VTD_FAULT_WRITE = 0x5, /* Address-translation fault,
non-writable */
+ VTD_FAULT_READ = 0x6, /* Address-translation fault,
non-readable */
+ VTD_FAULT_PTE_INVAL = 0x7, /* page table hw access error */
+ VTD_FAULT_ROOT_INVAL = 0x8, /* root table hw access error */
+ VTD_FAULT_CTX_TBL_INVAL = 0x9, /* context entry hw access error */
+ VTD_FAULT_ROOT_RESERVED = 0xa, /* non-zero reserved field in root
entry */
+ VTD_FAULT_CTX_RESERVED = 0xb, /* non-zero reserved field in context
entry */
+ VTD_FAULT_PTE_RESERVED = 0xc, /* non-zero reserved field in paging
entry */
+ VTD_FAULT_CTX_TT = 0xd, /* invalid translation type */
+};
+
+const char *vtd_faults[] = {
+ "Software",
+ "Root Entry Not Present",
+ "Context Entry Not Present",
+ "Context Entry Invalid",
+ "Address Beyond MGAW",
+ "Write",
+ "Read",
+ "Paging Entry Invalid",
+ "Root Table Invalid",
+ "Context Table Invalid",
+ "Root Entry Reserved",
+ "Context Entry Reserved",
+ "Paging Entry Reserved",
+ "Context Entry TT",
+ "Reserved",
+};
+
+void
+iommu_showcfg(struct iommu_softc *iommu)
+{
+ int i, j, sts, cmd;
+
+ cmd = iommu_readl(iommu, DMAR_GCMD_REG);
+ sts = iommu_readl(iommu, DMAR_GSTS_REG);
+ printf("iommu%d: flags:%d root pa:%.16llx %s %s %s %.8x %.8x\n",
+ iommu->id, iommu->flags, iommu_readq(iommu, DMAR_RTADDR_REG),
+ sts & GSTS_TES ? "enabled" : "disabled",
+ sts & GSTS_QIES ? "qi" : "ccmd",
+ sts & GSTS_IRES ? "ir" : "",
+ cmd, sts);
+ for (i = 0; i < 256; i++) {
+ if (!root_entry_is_valid(&iommu->root[i])) {
+ continue;
+ }
+#if 0
+ printf("iommu%d: bus %.2x %.16llx\n",
+ iommu->id, i, iommu->root[i].lo & ~0xFFF);
+#endif
+ for (j = 0; j < 256; j++) {
+ struct context_entry *ctx;
+
+ ctx = iommu->ctx[i] + j;
+ if (!context_entry_is_valid(ctx)) {
+ continue;
+ }
+ printf(" %.2x:%.2x.%x lvl:%d did:%.4x tt:%d
ptep:%.16llx\n",
+ i, (j>> 3), j & 7,
+ context_address_width(ctx),
+ context_domain_id(ctx),
+ context_translation_type(ctx),
+ context_pte(ctx));
+ }
+ }
+}
+
+void
+iommu_showfault(struct iommu_softc *iommu, int fri, struct fault_entry *fe)
+{
+ int bus, dev, fun, type, fr;
+
+ if (!(fe->hi & FRCD_HI_F))
+ return;
+ type = (fe->hi & FRCD_HI_T) ? 'r' : 'w';
+ fr = (fe->hi>> FRCD_HI_FR_SHIFT) & FRCD_HI_FR_MASK;
+ bus = (fe->hi>> FRCD_HI_BUS_SHIFT) & FRCD_HI_BUS_MASK;
+ dev = (fe->hi>> FRCD_HI_DEV_SHIFT) & FRCD_HI_DEV_MASK;
+ fun = (fe->hi>> FRCD_HI_FUN_SHIFT) & FRCD_HI_FUN_MASK;
+ printf("fri%d: dmar: %.2x:%.2x.%x %s error at %llx fr:%d [%s]
iommu:%d\n",
+ fri, bus, dev, fun,
+ type == 'r' ? "read" : "write",
+ fe->lo,
+ fr, fr <= 13 ? vtd_faults[fr] : "unknown",
+ iommu->id);
+ panic("error");
+}
Index: dev/acpi/acpidmar.h
===================================================================
RCS file: dev/acpi/acpidmar.h
diff -N dev/acpi/acpidmar.h
--- /dev/null 1 Jan 1970 00:00:00 -0000
+++ dev/acpi/acpidmar.h 22 Jul 2015 20:25:36 -0000
@@ -0,0 +1,447 @@
+/*
+ * Copyright (c) 2015 Jordan Hargrave <jordan_hargr...@hotmail.com>
+ *
+ * Permission to use, copy, modify, and distribute this software for any
+ * purpose with or without fee is hereby granted, provided that the above
+ * copyright notice and this permission notice appear in all copies.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
+ * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
+ * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
+ * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
+ * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
+ * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
+ * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+ */
+
+#ifndef _DEV_ACPI_DMARREG_H_
+#define _DEV_ACPI_DMARREG_H_
+
+#define VTD_STRIDE_MASK 0x1FF
+#define VTD_STRIDE_SIZE 9
+#define VTD_PAGE_SIZE 4096
+
+#define _xbit(x,y) (((x)>> (y)) & 1)
+#define _xfld(x,y) (uint32_t)(((x)>> y##_SHIFT) & y##_MASK)
+
+#define DMAR_VER_REG 0x00 /* 32:Arch version supported by this
IOMMU */
+#define DMAR_RTADDR_REG 0x20 /* 64:Root entry table */
+#define DMAR_FEDATA_REG 0x3c /* 32:Fault event interrupt
data register */
+#define DMAR_FEADDR_REG 0x40 /* 32:Fault event interrupt
addr register */
+#define DMAR_FEUADDR_REG 0x44 /* 32:Upper address register */
+#define DMAR_AFLOG_REG 0x58 /* 64:Advanced Fault control */
+#define DMAR_PMEN_REG 0x64 /* 32:Enable Protected Memory Region */
+#define DMAR_PLMBASE_REG 0x68 /* 32:PMRR Low addr */
+#define DMAR_PLMLIMIT_REG 0x6c /* 32:PMRR low limit */
+#define DMAR_PHMBASE_REG 0x70 /* 64:pmrr high base addr */
+#define DMAR_PHMLIMIT_REG 0x78 /* 64:pmrr high limit */
+#define DMAR_ICS_REG 0x9C /* 32:Invalidation complete status
register */
+#define DMAR_IECTL_REG 0xa0 /* 32:Invalidation event control
register */
+#define DMAR_IEDATA_REG 0xa4 /* 32:Invalidation event data
register */
+#define DMAR_IEADDR_REG 0xa8 /* 32:Invalidation event
address register */
+#define DMAR_IEUADDR_REG 0xac /* 32:Invalidation event upper address
register */
+#define DMAR_IRTA_REG 0xb8 /* 64:Interrupt remapping table addr
register */
+#define DMAR_CAP_REG 0x08 /* 64:Hardware supported capabilities */
+
+#define CAP_PI (1LL << 59)
+#define CAP_FL1GP (1LL << 56)
+#define CAP_DRD (1LL << 55)
+#define CAP_DWD (1LL << 54)
+#define CAP_MAMV_MASK 0x3F
+#define CAP_MAMV_SHIFT 48LL
+#define cap_mamv(x) _xfld(x,CAP_MAMV)
+#define CAP_NFR_MASK 0xFF
+#define CAP_NFR_SHIFT 40LL
+#define cap_nfr(x) (_xfld(x,CAP_NFR) + 1)
+#define CAP_PSI (1LL << 39)
+#define CAP_SLLPS_MASK 0xF
+#define CAP_SLLPS_SHIFT 34LL
+#define cap_sllps(x) _xfld(x,CAP_SLLPS)
+#define CAP_FRO_MASK 0x3FF
+#define CAP_FRO_SHIFT 24LL
+#define cap_fro(x) (_xfld(x,CAP_FRO) * 16)
+#define CAP_ZLR (1LL << 22)
+#define CAP_MGAW_MASK 0x3F
+#define CAP_MGAW_SHIFT 16LL
+#define cap_mgaw(x) (_xfld(x,CAP_MGAW) + 1)
+#define CAP_SAGAW_MASK 0x1F
+#define CAP_SAGAW_SHIFT 8LL
+#define cap_sagaw(x) _xfld(x,CAP_SAGAW)
+#define CAP_CM (1LL << 7)
+#define CAP_PHMR (1LL << 6)
+#define CAP_PLMR (1LL << 5)
+#define CAP_RWBF (1LL << 4)
+#define CAP_AFL (1LL << 3)
+#define CAP_ND_MASK 0x7
+#define CAP_ND_SHIFT 0x00
+#define cap_nd(x) (16 << (((x) & CAP_ND_MASK) << 1))
+
+#define DMAR_ECAP_REG 0x10 /* 64:Extended capabilities supported */
+#define ECAP_PSS_MASK 0x1F
+#define ECAP_PSS_SHIFT 35
+#define ECAP_EAFS (1LL << 34)
+#define ECAP_NWFS (1LL << 33)
+#define ECAP_SRS (1LL << 31)
+#define ECAP_ERS (1LL << 30)
+#define ECAP_PRS (1LL << 29)
+#define ECAP_PASID (1LL << 28)
+#define ECAP_DIS (1LL << 27)
+#define ECAP_NEST (1LL << 26)
+#define ECAP_MTS (1LL << 25)
+#define ECAP_ECS (1LL << 24)
+#define ECAP_MHMV_MASK 0xF
+#define ECAP_MHMV_SHIFT 0x20
+#define ecap_mhmv(x) _xfld(x,ECAP_MHMV)
+#define ECAP_IRO_MASK 0x3FF /* IOTLB Register */
+#define ECAP_IRO_SHIFT 0x8
+#define ecap_iro(x) (_xfld(x,ECAP_IRO) * 16)
+#define ECAP_SC (1LL << 7) /* Snoop Control */
+#define ECAP_PT (1LL << 6) /* HW Passthru */
+#define ECAP_EIM (1LL << 4)
+#define ECAP_IR (1LL << 3) /* Interrupt remap */
+#define ECAP_DT (1LL << 2) /* Device IOTLB */
+#define ECAP_QI (1LL << 1) /* Queued Invalidation */
+#define ECAP_C (1LL << 0) /* Coherent cache */
+
+#define DMAR_GCMD_REG 0x18 /* 32:Global command register */
+#define GCMD_TE (1LL << 31)
+#define GCMD_SRTP (1LL << 30)
+#define GCMD_SFL (1LL << 29)
+#define GCMD_EAFL (1LL << 28)
+#define GCMD_WBF (1LL << 27)
+#define GCMD_QIE (1LL << 26)
+#define GCMD_IRE (1LL << 25)
+#define GCMD_SIRTP (1LL << 24)
+#define GCMD_CFI (1LL << 23)
+
+#define DMAR_GSTS_REG 0x1c /* 32:Global status register */
+#define GSTS_TES (1LL << 31)
+#define GSTS_RTPS (1LL << 30)
+#define GSTS_FLS (1LL << 29)
+#define GSTS_AFLS (1LL << 28)
+#define GSTS_WBFS (1LL << 27)
+#define GSTS_QIES (1LL << 26)
+#define GSTS_IRES (1LL << 25)
+#define GSTS_IRTPS (1LL << 24)
+#define GSTS_CFIS (1LL << 23)
+
+#define DMAR_CCMD_REG 0x28 /* 64:Context command reg */
+#define CCMD_ICC (1LL << 63)
+#define CCMD_CIRG_MASK 0x3
+#define CCMD_CIRG_SHIFT 61
+#define CCMD_CIRG(x) ((uint64_t)(x) << CCMD_CIRG_SHIFT)
+#define CCMD_CAIG_MASK 0x3
+#define CCMD_CAIG_SHIFT 59
+#define CCMD_FM_MASK 0x3
+#define CCMD_FM_SHIFT 32
+#define CCMD_FM(x) (((uint64_t)(x) << CCMD_FM_SHIFT))
+#define CCMD_SID_MASK 0xFFFF
+#define CCMD_SID_SHIFT 8
+#define CCMD_SID(x) (((x) << CCMD_SID_SHIFT))
+#define CCMD_DID_MASK 0xFFFF
+#define CCMD_DID_SHIFT 0
+#define CCMD_DID(x) (((x) << CCMD_DID_SHIFT))
+
+#define CIG_GLOBAL CCMD_CIRG(CTX_GLOBAL)
+#define CIG_DOMAIN CCMD_CIRG(CTX_DOMAIN)
+#define CIG_DEVICE CCMD_CIRG(CTX_DEVICE)
+
+
+#define DMAR_FSTS_REG 0x34 /* 32:Fault Status register */
+#define FSTS_FRI_MASK 0xFF
+#define FSTS_FRI_SHIFT 8
+#define FSTS_PRO (1LL << 7)
+#define FSTS_ITE (1LL << 6)
+#define FSTS_ICE (1LL << 5)
+#define FSTS_IQE (1LL << 4)
+#define FSTS_APF (1LL << 3)
+#define FSTS_APO (1LL << 2)
+#define FSTS_PPF (1LL << 1)
+#define FSTS_PFO (1LL << 0)
+
+#define DMAR_FECTL_REG 0x38 /* 32:Fault control register */
+#define FECTL_IM (1LL << 31)
+#define FECTL_IP (1LL << 30)
+
+#define FRCD_HI_F (1LL << (127-64))
+#define FRCD_HI_T (1LL << (126-64))
+#define FRCD_HI_AT_MASK 0x3
+#define FRCD_HI_AT_SHIFT (124-64)
+#define FRCD_HI_PV_MASK 0xFFFFF
+#define FRCD_HI_PV_SHIFT (104-64)
+#define FRCD_HI_FR_MASK 0xFF
+#define FRCD_HI_FR_SHIFT (96-64)
+#define FRCD_HI_PP (1LL << (95-64))
+
+#define FRCD_HI_SID_MASK 0xFF
+#define FRCD_HI_SID_SHIFT 0
+#define FRCD_HI_BUS_SHIFT 8
+#define FRCD_HI_BUS_MASK 0xFF
+#define FRCD_HI_DEV_SHIFT 3
+#define FRCD_HI_DEV_MASK 0x1F
+#define FRCD_HI_FUN_SHIFT 0
+#define FRCD_HI_FUN_MASK 0x7
+
+#define DMAR_IOTLB_REG(x) (ecap_iro((x)->ecap) + 8)
+#define DMAR_IVA_REG(x) (ecap_iro((x)->ecap) + 0)
+
+#define DMAR_FRIH_REG(x,i) (cap_fro((x)->cap) + 16*(i) + 8)
+#define DMAR_FRIL_REG(x,i) (cap_fro((x)->cap) + 16*(i) + 0)
+
+#define IOTLB_IVT (1LL << 63)
+#define IOTLB_IIRG_MASK 0x3
+#define IOTLB_IIRG_SHIFT 60
+#define IOTLB_IIRG(x) ((uint64_t)(x) << IOTLB_IIRG_SHIFT)
+#define IOTLB_IAIG_MASK 0x3
+#define IOTLB_IAIG_SHIFT 57
+#define IOTLB_DR (1LL << 49)
+#define IOTLB_DW (1LL << 48)
+#define IOTLB_DID_MASK 0xFFFF
+#define IOTLB_DID_SHIFT 32
+#define IOTLB_DID(x) ((uint64_t)(x) << IOTLB_DID_SHIFT)
+
+#define IIG_GLOBAL IOTLB_IIRG(IOTLB_GLOBAL)
+#define IIG_DOMAIN IOTLB_IIRG(IOTLB_DOMAIN)
+#define IIG_PAGE IOTLB_IIRG(IOTLB_PAGE)
+
+#define DMAR_IQH_REG 0x80 /* 64:Invalidation queue head register */
+#define DMAR_IQT_REG 0x88 /* 64:Invalidation queue tail register */
+#define DMAR_IQA_REG 0x90 /* 64:Invalidation queue addr register */
+#define IQA_QS_256 0 /* 256 entries */
+#define IQA_QS_512 1 /* 512 */
+#define IQA_QS_1K 2 /* 1024 */
+#define IQA_QS_2K 3 /* 2048 */
+#define IQA_QS_4K 4 /* 4096 */
+#define IQA_QS_8K 5 /* 8192 */
+#define IQA_QS_16K 6 /* 16384 */
+#define IQA_QS_32K 7 /* 32768 */
+
+/*
+ * Root Entry: one per bus (256 x 128 bit = 4k)
+ * 0 = Present
+ * 1:11 = Reserved
+ * 12:HAW-1 = Context Table Pointer
+ * HAW:63 = Reserved
+ * 64:127 = Reserved
+ */
+#define ROOT_P (1L << 0)
+struct root_entry {
+ uint64_t lo;
+ uint64_t hi;
+};
+
+/* Check if root entry is valid */
+static inline bool
+root_entry_is_valid(struct root_entry *re)
+{
+ return (re->lo & ROOT_P);
+}
+
+/*
+ * Context Entry: one per devfn (256 x 128 bit = 4k)
+ * 0 = Present
+ * 1 = Fault Processing Disable
+ * 2:3 = Translation Type
+ * 4:11 = Reserved
+ * 12:63 = Second Level Page Translation
+ * 64:66 = Address Width (# PTE levels)
+ * 67:70 = Ignore
+ * 71 = Reserved
+ * 72:87 = Domain ID
+ * 88:127 = Reserved
+ */
+#define CTX_P (1L << 0)
+#define CTX_FPD (1L << 1)
+#define CTX_T_MASK 0x3
+#define CTX_T_SHIFT 2
+enum {
+ CTX_T_MULTI,
+ CTX_T_IOTLB,
+ CTX_T_PASSTHRU
+};
+
+#define CTX_H_AW_MASK 0x7
+#define CTX_H_AW_SHIFT 0
+#define CTX_H_DID_MASK 0xFFFF
+#define CTX_H_DID_SHIFT 8
+
+struct context_entry {
+ uint64_t lo;
+ uint64_t hi;
+};
+
+/* Set fault processing enable/disable */
+static inline void
+context_set_fpd(struct context_entry *ce, int enable)
+{
+ ce->lo &= ~CTX_FPD;
+ if (enable)
+ ce->lo |= CTX_FPD;
+}
+
+/* Set context entry present */
+static inline void
+context_set_present(struct context_entry *ce)
+{
+ ce->lo |= CTX_P;
+}
+
+/* Set Second Level Page Table Entry PA */
+static inline void
+context_set_slpte(struct context_entry *ce, paddr_t slpte)
+{
+ ce->lo &= 0xFFF;
+ ce->lo |= (slpte & ~0xFFF);
+}
+
+/* Set translation type */
+static inline void
+context_set_translation_type(struct context_entry *ce, int tt)
+{
+ ce->lo &= ~(CTX_T_MASK << CTX_T_SHIFT);
+ ce->lo |= ((tt & CTX_T_MASK) << CTX_T_SHIFT);
+}
+
+/* Set Address Width (# of Page Table levels) */
+static inline void
+context_set_address_width(struct context_entry *ce, int lvl)
+{
+ ce->hi &= ~(CTX_H_AW_MASK << CTX_H_AW_SHIFT);
+ ce->hi |= ((lvl & CTX_H_AW_MASK) << CTX_H_AW_SHIFT);
+}
+
+/* Set domain ID */
+static inline void
+context_set_domain_id(struct context_entry *ce, int did)
+{
+ ce->hi &= ~(CTX_H_DID_MASK << CTX_H_DID_SHIFT);
+ ce->hi |= ((did & CTX_H_DID_MASK) << CTX_H_DID_SHIFT);
+}
+
+/* Get Second Level Page Table PA */
+static inline uint64_t
+context_pte(struct context_entry *ce)
+{
+ return (ce->lo & ~0xFFF);
+}
+
+/* Get translation type */
+static inline int
+context_translation_type(struct context_entry *ce)
+{
+ return (ce->lo>> CTX_T_SHIFT) & CTX_T_MASK;
+}
+
+/* Get domain ID */
+static inline int
+context_domain_id(struct context_entry *ce)
+{
+ return (ce->hi>> CTX_H_DID_SHIFT) & CTX_H_DID_MASK;
+}
+
+/* Get Address Width */
+static inline int
+context_address_width(struct context_entry *ce)
+{
+ return 30 + ((ce->hi>> CTX_H_AW_SHIFT) & CTX_H_AW_MASK) *
+ VTD_STRIDE_SIZE;
+}
+
+/* Check if context entry is valid */
+static inline bool
+context_entry_is_valid(struct context_entry *ce)
+{
+ return (ce->lo & CTX_P);
+}
+
+/*
+ * Fault entry
+ * 0..HAW-1 = Fault address
+ * HAW:63 = Reserved
+ * 64:71 = Source ID
+ * 96:103 = Fault Reason
+ * 104:123 = PV
+ * 124:125 = Address Translation type
+ * 126 = Type (0 = Read, 1 = Write)
+ * 127 = Fault bit
+ */
+struct fault_entry
+{
+ uint64_t lo;
+ uint64_t hi;
+};
+
+/* PTE Entry: 512 x 64-bit = 4k */
+#define PTE_P (1L << 0)
+#define PTE_R 0x00
+#define PTE_W (1L << 1)
+struct pte_entry {
+ uint64_t val;
+};
+
+/*
+ * Queued Invalidation entry
+ * 0:3 = 01h
+ * 4:5 = Granularity
+ * 6:15 = Reserved
+ * 16:31 = Domain ID
+ * 32:47 = Source ID
+ * 48:49 = FM
+ */
+
+/* Invalidate Context Entry */
+#define QI_CTX_DID_MASK 0xFFFF
+#define QI_CTX_DID_SHIFT 16
+#define QI_CTX_SID_MASK 0xFFFF
+#define QI_CTX_SID_SHIFT 32
+#define QI_CTX_FM_MASK 0x3
+#define QI_CTX_FM_SHIFT 48
+#define QI_CTX_IG_MASK 0x3
+#define QI_CTX_IG_SHIFT 4
+#define QI_CTX_DID(x) (((uint64_t)(x) << QI_CTX_DID_SHIFT))
+#define QI_CTX_SID(x) (((uint64_t)(x) << QI_CTX_SID_SHIFT))
+#define QI_CTX_FM(x) (((uint64_t)(x) << QI_CTX_FM_SHIFT))
+
+#define QI_CTX_IG_GLOBAL (CTX_GLOBAL << QI_CTX_IG_SHIFT)
+#define QI_CTX_IG_DOMAIN (CTX_DOMAIN << QI_CTX_IG_SHIFT)
+#define QI_CTX_IG_DEVICE (CTX_DEVICE << QI_CTX_IG_SHIFT)
+
+/* Invalidate IOTLB Entry */
+#define QI_IOTLB_DID_MASK 0xFFFF
+#define QI_IOTLB_DID_SHIFT 16
+#define QI_IOTLB_IG_MASK 0x3
+#define QI_IOTLB_IG_SHIFT 4
+#define QI_IOTLB_DR (1LL << 6)
+#define QI_IOTLB_DW (1LL << 5)
+#define QI_IOTLB_DID(x) (((uint64_t)(x) << QI_IOTLB_DID_SHIFT))
+
+#define QI_IOTLB_IG_GLOBAL (1 << QI_IOTLB_IG_SHIFT)
+#define QI_IOTLB_IG_DOMAIN (2 << QI_IOTLB_IG_SHIFT)
+#define QI_IOTLB_IG_PAGE (3 << QI_IOTLB_IG_SHIFT)
+
+/* QI Commands */
+#define QI_CTX 0x1
+#define QI_IOTLB 0x2
+#define QI_DEVTLB 0x3
+#define QI_INTR 0x4
+#define QI_WAIT 0x5
+#define QI_EXTTLB 0x6
+#define QI_PAS 0x7
+#define QI_EXTDEV 0x8
+
+struct qi_entry {
+ uint64_t lo;
+ uint64_t hi;
+};
+
+enum {
+ CTX_GLOBAL = 1,
+ CTX_DOMAIN,
+ CTX_DEVICE,
+
+ IOTLB_GLOBAL = 1,
+ IOTLB_DOMAIN,
+ IOTLB_PAGE,
+};
+
+#endif
Index: dev/acpi/files.acpi
===================================================================
RCS file: /cvs/src/sys/dev/acpi/files.acpi,v
retrieving revision 1.27
diff -u -p -u -p -r1.27 files.acpi
--- dev/acpi/files.acpi 6 Nov 2013 10:40:36 -0000 1.27
+++ dev/acpi/files.acpi 22 Jul 2015 20:25:36 -0000
@@ -56,6 +56,11 @@ device acpimadt
attach acpimadt at acpi
file dev/acpi/acpimadt.c acpimadt
+# DMA Remapping Table
+device acpidmar
+attach acpidmar at acpi
+file dev/acpi/acpidmar.c acpidmar
+
# Memory Mapped Configuration Space Address Description Table
device acpimcfg
attach acpimcfg at acpi
