On Mon, Jun 24, 2019 at 01:22:55AM -0400, Yan Zhao wrote:
> On Thu, Jun 20, 2019 at 09:04:43PM +0800, Peter Xu wrote:
> > On Thu, Jun 20, 2019 at 08:59:55PM +0800, Peter Xu wrote:
> > > On Thu, Jun 20, 2019 at 10:35:29AM +0200, Paolo Bonzini wrote:
> > > > On 20/06/19 06:02, Peter Xu wrote:
> > > > > Seems workable, to be explicit - we can even cut it into chunks with
> > > > > different size to be efficient.
> > > >
> > > > Yes, this is not hard (completely untested):
> > > >
> > > > diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
> > > > index 44b1231157..541538bc6c 100644
> > > > --- a/hw/i386/intel_iommu.c
> > > > +++ b/hw/i386/intel_iommu.c
> > > > @@ -3388,39 +3388,34 @@ static void
> > > > vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
> > > > }
> > > >
> > > > assert(start <= end);
> > > > - size = end - start;
> > > > + while (end > start) {
> > > > + size = end - start;
> > > > + /* Only keep the lowest bit of either size or start. */
> > > > + size = MIN(size & -size, start & -start);
> > >
> > > I feel like this can be problematic. I'm imaging:
> > >
> > > start=0x1000_0000, size=0x1000_1000
> > >
> > > This will get size=0x1000 but actually we can do size=0x1000_0000 as
> > > the first.
> > >
> > > > + /* Should not happen, but limit to address width too just in
> > > > case */
> > > > + size = MIN(size, 1ULL << s->aw_bits);
> > > >
> > > > - if (ctpop64(size) != 1) {
> > > > - /*
> > > > - * This size cannot format a correct mask. Let's enlarge it to
> > > > - * suite the minimum available mask.
> > > > - */
> > > > - int n = 64 - clz64(size);
> > > > - if (n > s->aw_bits) {
> > > > - /* should not happen, but in case it happens, limit it */
> > > > - n = s->aw_bits;
> > > > - }
> > > > - size = 1ULL << n;
> > > > - }
> > > > + assert((start & (size - 1)) == 0);
> > > >
> > > > - entry.target_as = &address_space_memory;
> > > > - /* Adjust iova for the size */
> > > > - entry.iova = n->start & ~(size - 1);
> > > > - /* This field is meaningless for unmap */
> > > > - entry.translated_addr = 0;
> > > > - entry.perm = IOMMU_NONE;
> > > > - entry.addr_mask = size - 1;
> > > > + entry.target_as = &address_space_memory;
> > > > + entry.iova = start;
> > > > + /* This field is meaningless for unmap */
> > > > + entry.translated_addr = 0;
> > > > + entry.perm = IOMMU_NONE;
> > > > + entry.addr_mask = size - 1;
> > >
> > > (some of the fields can be moved out of loop because they are
> > > constants)
> > >
> > > >
> > > > - trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
> > > > - VTD_PCI_SLOT(as->devfn),
> > > > - VTD_PCI_FUNC(as->devfn),
> > > > - entry.iova, size);
> > > > + trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
> > > > + VTD_PCI_SLOT(as->devfn),
> > > > + VTD_PCI_FUNC(as->devfn),
> > > > + entry.iova, size);
> > >
> > > Can move this out because this is a trace only so we don't have
> > > restriction on mask?
> > >
> > > >
> > > > - map.iova = entry.iova;
> > > > - map.size = entry.addr_mask;
> > > > - iova_tree_remove(as->iova_tree, &map);
> > > > + map.iova = entry.iova;
> > > > + map.size = entry.addr_mask;
> > > > + iova_tree_remove(as->iova_tree, &map);
> > >
> > > Same here?
> > >
> > > >
> > > > - memory_region_notify_one(n, &entry);
> > > > + memory_region_notify_one(n, &entry);
> > > > + start += size;
> > > > + }
> > > > }
> > > >
> > > > static void vtd_address_space_unmap_all(IntelIOMMUState *s)
> > > >
> > > >
> > > > Yan,
> > > >
> > > > if something like this works for you, let me know and I will submit it
> > > > as a proper patch.
> > > >
> > > > Paolo
> > >
> > > Since during review I'm thinking how to generate a correct sequence of
> > > these masks... here's my try below with above issues fixed... :)
> > >
> > > I've tried compile but not tested. Yan can test it, or I can do it
> > > too tomorrow after I find some machines.
> > >
> > > Thanks,
> > >
> > > ------------------------------------------------------------
> > > diff --git a/hw/i386/intel_iommu.c b/hw/i386/intel_iommu.c
> > > index 44b1231157..cfbd225f0a 100644
> > > --- a/hw/i386/intel_iommu.c
> > > +++ b/hw/i386/intel_iommu.c
> > > @@ -3363,11 +3363,32 @@ VTDAddressSpace *vtd_find_add_as(IntelIOMMUState
> > > *s, PCIBus *bus, int devfn)
> > > return vtd_dev_as;
> > > }
> > >
> > > +static uint64_t vtd_get_next_mask(uint64_t start, uint64_t size, int gaw)
> > > +{
> > > + /* Tries to find smallest mask from start first */
> > > + uint64_t rmask = start & -start, max_mask = 1ULL << gaw;
> > > +
> > > + assert(size && gaw > 0 && gaw < 64);
> > > +
> > > + /* Zero start, or too big */
> > > + if (!rmask || rmask > max_mask) {
> > > + rmask = max_mask;
> > > + }
> > > +
> > > + /* If the start mask worked, then use it */
> > > + if (rmask <= size) {
> > > + return rmask;
> > > + }
> > > +
> > > + /* Find the largest page mask from size */
> > > + return 1ULL << (63 - clz64(size));
> > > +}
> > > +
> > > /* Unmap the whole range in the notifier's scope. */
> > > static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier
> > > *n)
> > > {
> > > IOMMUTLBEntry entry;
> > > - hwaddr size;
> > > + hwaddr size, remain;
> > > hwaddr start = n->start;
> > > hwaddr end = n->end;
> > > IntelIOMMUState *s = as->iommu_state;
> > > @@ -3388,39 +3409,28 @@ static void
> > > vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
> > > }
> > >
> > > assert(start <= end);
> > > - size = end - start;
> > > -
> > > - if (ctpop64(size) != 1) {
> > > - /*
> > > - * This size cannot format a correct mask. Let's enlarge it to
> > > - * suite the minimum available mask.
> > > - */
> > > - int n = 64 - clz64(size);
> > > - if (n > s->aw_bits) {
> > > - /* should not happen, but in case it happens, limit it */
> > > - n = s->aw_bits;
> > > - }
> > > - size = 1ULL << n;
> > > - }
> > > -
> > > + size = remain = end - start;
> > > entry.target_as = &address_space_memory;
> > > - /* Adjust iova for the size */
> > > - entry.iova = n->start & ~(size - 1);
> > > + entry.perm = IOMMU_NONE;
> > > /* This field is meaningless for unmap */
> > > entry.translated_addr = 0;
> > > - entry.perm = IOMMU_NONE;
> > > - entry.addr_mask = size - 1;
> > > +
> > > + while (remain) {
> > > + uint64_t mask = vtd_get_next_mask(start, remain, s->aw_bits);
> > > +
> > > + entry.iova = start;
> > > + entry.addr_mask = mask - 1;
> > > + memory_region_notify_one(n, &entry);
> >
> > Sorry, I at least missed these lines:
> >
> > start += mask;
> > remain -= mask;
> >
> > > + }
> > >
> > > trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
> > > VTD_PCI_SLOT(as->devfn),
> > > VTD_PCI_FUNC(as->devfn),
> > > - entry.iova, size);
> > > + n->start, size);
> > >
> > > - map.iova = entry.iova;
> > > - map.size = entry.addr_mask;
> > > + map.iova = n->start;
> > > + map.size = size;
> > > iova_tree_remove(as->iova_tree, &map);
> > > -
> > > - memory_region_notify_one(n, &entry);
> > > }
> > >
> > > static void vtd_address_space_unmap_all(IntelIOMMUState *s)
> > > ------------------------------------------------------------
> > >
> > > Regards,
> > >
> > > --
> > > Peter Xu
> >
> > Regards,
> >
> > --
> > Peter Xu
>
> hi Peter and Paolo,
> I tested with code and it's fine in my side.
> It's base on your version with some minor modifications, such as size is
> now (end - start + 1) now.
> Thanks
> Yan
Hi, Yan,
Thanks for testing the patches. I think below change [1] is not
related to the problem so I tend to split it out. For [2] I'll change
to an assertion if you won't disagree. I'll reorganize the patches
and post a formal version with proper authorships soon.
Thanks,
>
> +static uint64_t vtd_get_next_mask(uint64_t start, uint64_t size, int gaw)
> +{
> + /* Tries to find smallest mask from start first */
> + uint64_t rmask = start & -start, max_mask = 1ULL << gaw;
> + assert(size && gaw > 0 && gaw < 64);
> + /* Zero start, or too big */
> + if (!rmask || rmask > max_mask) {
> + rmask = max_mask;
> + }
> + /* If the start mask worked, then use it */
> + if (rmask <= size) {
> + return rmask;
> + }
> +
> + /* Find the largest page mask from size */
> + return 1ULL << (63 - clz64(size));
> +}
> +
> /* Unmap the whole range in the notifier's scope. */
> static void vtd_address_space_unmap(VTDAddressSpace *as, IOMMUNotifier *n)
> {
> IOMMUTLBEntry entry;
> - hwaddr size;
> + hwaddr size, remain;
> hwaddr start = n->start;
> hwaddr end = n->end;
> IntelIOMMUState *s = as->iommu_state;
> @@ -3380,48 +3398,46 @@ static void vtd_address_space_unmap(VTDAddressSpace
> *as, IOMMUNotifier *n)
> * VT-d spec), otherwise we need to consider overflow of 64 bits.
> */
>
> - if (end > VTD_ADDRESS_SIZE(s->aw_bits)) {
> + if (end > VTD_ADDRESS_SIZE(s->aw_bits) - 1) {
> /*
> * Don't need to unmap regions that is bigger than the whole
> * VT-d supported address space size
> */
> - end = VTD_ADDRESS_SIZE(s->aw_bits);
> + end = VTD_ADDRESS_SIZE(s->aw_bits) - 1;
[1]
> }
>
> assert(start <= end);
> - size = end - start;
>
> - if (ctpop64(size) != 1) {
> - /*
> - * This size cannot format a correct mask. Let's enlarge it to
> - * suite the minimum available mask.
> - */
> - int n = 64 - clz64(size);
> - if (n > s->aw_bits) {
> - /* should not happen, but in case it happens, limit it */
> - n = s->aw_bits;
> - }
> - size = 1ULL << n;
> - }
> + size = remain = end - start + 1;
>
> entry.target_as = &address_space_memory;
> - /* Adjust iova for the size */
> - entry.iova = n->start & ~(size - 1);
> +
> + entry.perm = IOMMU_NONE;
> /* This field is meaningless for unmap */
> entry.translated_addr = 0;
> - entry.perm = IOMMU_NONE;
> - entry.addr_mask = size - 1;
> +
> + while (remain >= VTD_PAGE_SIZE) {
> + uint64_t mask = vtd_get_next_mask(start, remain, s->aw_bits);
> +
> + entry.iova = start;
> + entry.addr_mask = mask - 1;
> + memory_region_notify_one(n, &entry);
> + start += mask;
> + remain -= mask;
> + }
> +
> + if (remain) {
> + warn_report("Unmapping unaligned range %lx-%lx", start, end);
[2]
> + }
>
> trace_vtd_as_unmap_whole(pci_bus_num(as->bus),
> VTD_PCI_SLOT(as->devfn),
> VTD_PCI_FUNC(as->devfn),
> - entry.iova, size);
> -
> - map.iova = entry.iova;
> - map.size = entry.addr_mask;
> + n->start, size);
> + map.iova = n->start;
> + map.size = size;
> iova_tree_remove(as->iova_tree, &map);
>
> - memory_region_notify_one(n, &entry);
> }
>
> static void vtd_address_space_unmap_all(IntelIOMMUState *s)
> --
> 2.7.4
>
>
>
Regards,
--
Peter Xu
