Use iommu_iova_to_phys_length() to get PTE page size in batch_from_domain and raw_pages_from_domain, allowing traversal by actual mapping granularity instead of PAGE_SIZE steps.
Signed-off-by: Guanghui Feng <[email protected]> Acked-by: Shiqiang Zhang <[email protected]> Acked-by: Simon Guo <[email protected]> --- drivers/iommu/iommufd/pages.c | 74 +++++++++++++++++++++++++++++------ 1 file changed, 62 insertions(+), 12 deletions(-) diff --git a/drivers/iommu/iommufd/pages.c b/drivers/iommu/iommufd/pages.c index 9bdb2945afe1..40a2fe9adf9c 100644 --- a/drivers/iommu/iommufd/pages.c +++ b/drivers/iommu/iommufd/pages.c @@ -417,17 +417,42 @@ static void batch_from_domain(struct pfn_batch *batch, if (start_index == iopt_area_index(area)) page_offset = area->page_offset; while (start_index <= last_index) { + size_t pgsize; + unsigned long npages; + unsigned long i; + /* - * This is pretty slow, it would be nice to get the page size - * back from the driver, or have the driver directly fill the - * batch. + * Use iova_to_phys_length to get both the physical address + * and the contiguous mapped length in a single page table + * walk, allowing us to skip ahead by the contiguous region + * size instead of walking page tables for every PAGE_SIZE. + * Query at page-aligned iova so pgsize covers from page start. */ - phys = iommu_iova_to_phys(domain, iova) - page_offset; - if (!batch_add_pfn(batch, PHYS_PFN(phys))) - return; - iova += PAGE_SIZE - page_offset; + phys = iommu_iova_to_phys_length(domain, iova - page_offset, + &pgsize); + if (WARN_ON(phys == PHYS_ADDR_MAX)) + break; + if (WARN_ON(!pgsize || pgsize < PAGE_SIZE)) + pgsize = PAGE_SIZE; + + /* + * pgsize is the contiguous length from the page-aligned + * iova, so npages is simply pgsize / PAGE_SIZE. + */ + npages = pgsize / PAGE_SIZE; + npages = min_t(unsigned long, npages, + last_index - start_index + 1); + if (!npages) + npages = 1; + + for (i = 0; i < npages; i++) { + if (!batch_add_pfn(batch, PHYS_PFN(phys) + i)) + return; + } + + iova += npages * PAGE_SIZE - page_offset; page_offset = 0; - start_index++; + start_index += npages; } } @@ -445,11 +470,36 @@ static struct page **raw_pages_from_domain(struct iommu_domain *domain, if (start_index == iopt_area_index(area)) page_offset = area->page_offset; while (start_index <= last_index) { - phys = iommu_iova_to_phys(domain, iova) - page_offset; - *(out_pages++) = pfn_to_page(PHYS_PFN(phys)); - iova += PAGE_SIZE - page_offset; + size_t pgsize; + unsigned long npages; + unsigned long i; + + /* + * Resolve the contiguous mapped length together with the + * physical address so we can fill multiple struct page + * pointers per page table walk when the IOMMU uses large + * pages. Query at page-aligned iova so pgsize covers from + * page start. + */ + phys = iommu_iova_to_phys_length(domain, iova - page_offset, + &pgsize); + if (WARN_ON(phys == PHYS_ADDR_MAX)) + break; + if (WARN_ON(!pgsize || pgsize < PAGE_SIZE)) + pgsize = PAGE_SIZE; + + npages = pgsize / PAGE_SIZE; + npages = min_t(unsigned long, npages, + last_index - start_index + 1); + if (!npages) + npages = 1; + + for (i = 0; i < npages; i++) + *(out_pages++) = pfn_to_page(PHYS_PFN(phys) + i); + + iova += npages * PAGE_SIZE - page_offset; page_offset = 0; - start_index++; + start_index += npages; } return out_pages; } -- 2.43.7
