From e398fb4bdf522abdeb8ddb61235bbf66065fc105 Mon Sep 17 00:00:00 2001 From: Souptick Joarder Date: Sun, 12 Jul 2020 09:09:53 +0530 Subject: [PATCH 01/19] xen/privcmd: Corrected error handling path Previously, if lock_pages() end up partially mapping pages, it used to return -ERRNO due to which unlock_pages() have to go through each pages[i] till *nr_pages* to validate them. This can be avoided by passing correct number of partially mapped pages & -ERRNO separately, while returning from lock_pages() due to error. With this fix unlock_pages() doesn't need to validate pages[i] till *nr_pages* for error scenario and few condition checks can be ignored. Signed-off-by: Souptick Joarder Reviewed-by: Juergen Gross Reviewed-by: Paul Durrant Cc: John Hubbard Cc: Boris Ostrovsky Cc: Paul Durrant Link: https://lore.kernel.org/r/1594525195-28345-2-git-send-email-jrdr.linux@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/privcmd.c | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index a250d118144a..a9b2baa2c7be 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -580,13 +580,13 @@ static long privcmd_ioctl_mmap_batch( static int lock_pages( struct privcmd_dm_op_buf kbufs[], unsigned int num, - struct page *pages[], unsigned int nr_pages) + struct page *pages[], unsigned int nr_pages, unsigned int *pinned) { unsigned int i; for (i = 0; i < num; i++) { unsigned int requested; - int pinned; + int page_count; requested = DIV_ROUND_UP( offset_in_page(kbufs[i].uptr) + kbufs[i].size, @@ -594,14 +594,15 @@ static int lock_pages( if (requested > nr_pages) return -ENOSPC; - pinned = get_user_pages_fast( + page_count = get_user_pages_fast( (unsigned long) kbufs[i].uptr, requested, FOLL_WRITE, pages); - if (pinned < 0) - return pinned; + if (page_count < 0) + return page_count; - nr_pages -= pinned; - pages += pinned; + *pinned += page_count; + nr_pages -= page_count; + pages += page_count; } return 0; @@ -611,13 +612,8 @@ static void unlock_pages(struct page *pages[], unsigned int nr_pages) { unsigned int i; - if (!pages) - return; - - for (i = 0; i < nr_pages; i++) { - if (pages[i]) - put_page(pages[i]); - } + for (i = 0; i < nr_pages; i++) + put_page(pages[i]); } static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) @@ -630,6 +626,7 @@ static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) struct xen_dm_op_buf *xbufs = NULL; unsigned int i; long rc; + unsigned int pinned = 0; if (copy_from_user(&kdata, udata, sizeof(kdata))) return -EFAULT; @@ -683,9 +680,11 @@ static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) goto out; } - rc = lock_pages(kbufs, kdata.num, pages, nr_pages); - if (rc) + rc = lock_pages(kbufs, kdata.num, pages, nr_pages, &pinned); + if (rc < 0) { + nr_pages = pinned; goto out; + } for (i = 0; i < kdata.num; i++) { set_xen_guest_handle(xbufs[i].h, kbufs[i].uptr); From a0c34d225183fd4aca61dccb50e5783fa2f26db1 Mon Sep 17 00:00:00 2001 From: Souptick Joarder Date: Sun, 12 Jul 2020 09:09:54 +0530 Subject: [PATCH 02/19] xen/privcmd: Mark pages as dirty pages need to be marked as dirty before unpinned it in unlock_pages() which was oversight. This is fixed now. Signed-off-by: Souptick Joarder Suggested-by: John Hubbard Reviewed-by: Juergen Gross Reviewed-by: Paul Durrant Cc: John Hubbard Cc: Boris Ostrovsky Cc: Paul Durrant Link: https://lore.kernel.org/r/1594525195-28345-3-git-send-email-jrdr.linux@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/privcmd.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index a9b2baa2c7be..aa8ca58e6efc 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -612,8 +612,11 @@ static void unlock_pages(struct page *pages[], unsigned int nr_pages) { unsigned int i; - for (i = 0; i < nr_pages; i++) + for (i = 0; i < nr_pages; i++) { + if (!PageDirty(pages[i])) + set_page_dirty_lock(pages[i]); put_page(pages[i]); + } } static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) From ff669aa8124035a432d2b7eecdd876b308cd48ee Mon Sep 17 00:00:00 2001 From: Souptick Joarder Date: Sun, 12 Jul 2020 09:09:55 +0530 Subject: [PATCH 03/19] xen/privcmd: Convert get_user_pages*() to pin_user_pages*() In 2019, we introduced pin_user_pages*() and now we are converting get_user_pages*() to the new API as appropriate. [1] & [2] could be referred for more information. This is case 5 as per document [1]. [1] Documentation/core-api/pin_user_pages.rst [2] "Explicit pinning of user-space pages": https://lwn.net/Articles/807108/ Signed-off-by: Souptick Joarder Reviewed-by: Juergen Gross Reviewed-by: Paul Durrant Cc: John Hubbard Cc: Boris Ostrovsky Cc: Paul Durrant Link: https://lore.kernel.org/r/1594525195-28345-4-git-send-email-jrdr.linux@gmail.com Signed-off-by: Juergen Gross --- drivers/xen/privcmd.c | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/drivers/xen/privcmd.c b/drivers/xen/privcmd.c index aa8ca58e6efc..095d683ad574 100644 --- a/drivers/xen/privcmd.c +++ b/drivers/xen/privcmd.c @@ -594,7 +594,7 @@ static int lock_pages( if (requested > nr_pages) return -ENOSPC; - page_count = get_user_pages_fast( + page_count = pin_user_pages_fast( (unsigned long) kbufs[i].uptr, requested, FOLL_WRITE, pages); if (page_count < 0) @@ -610,13 +610,7 @@ static int lock_pages( static void unlock_pages(struct page *pages[], unsigned int nr_pages) { - unsigned int i; - - for (i = 0; i < nr_pages; i++) { - if (!PageDirty(pages[i])) - set_page_dirty_lock(pages[i]); - put_page(pages[i]); - } + unpin_user_pages_dirty_lock(pages, nr_pages, true); } static long privcmd_ioctl_dm_op(struct file *file, void __user *udata) From e5a52fd2b8cdb700b3c07b030e050a49ef3156b9 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sat, 18 Jul 2020 17:33:17 -0700 Subject: [PATCH 04/19] xen/gntdev: gntdev.h: drop a duplicated word Drop the repeated word "of" in a comment. Signed-off-by: Randy Dunlap Reviewed-by: Juergen Gross Cc: Boris Ostrovsky Cc: Juergen Gross Cc: xen-devel@lists.xenproject.org Link: https://lore.kernel.org/r/20200719003317.21454-1-rdunlap@infradead.org Signed-off-by: Juergen Gross --- include/uapi/xen/gntdev.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/uapi/xen/gntdev.h b/include/uapi/xen/gntdev.h index fe4423e518c6..9ac5515b9bc2 100644 --- a/include/uapi/xen/gntdev.h +++ b/include/uapi/xen/gntdev.h @@ -66,7 +66,7 @@ struct ioctl_gntdev_map_grant_ref { /* * Removes the grant references from the mapping table of an instance of - * of gntdev. N.B. munmap() must be called on the relevant virtual address(es) + * gntdev. N.B. munmap() must be called on the relevant virtual address(es) * before this ioctl is called, or an error will result. */ #define IOCTL_GNTDEV_UNMAP_GRANT_REF \ From 4e722d4fe27899f90491065edb0bb0a757f0b955 Mon Sep 17 00:00:00 2001 From: Randy Dunlap Date: Sat, 25 Jul 2020 17:17:31 -0700 Subject: [PATCH 05/19] xen: hypercall.h: fix duplicated word Change the repeated word "as" to "as a". Signed-off-by: Randy Dunlap Reviewed-by: Juergen Gross Cc: Boris Ostrovsky Cc: Juergen Gross Cc: xen-devel@lists.xenproject.org Link: https://lore.kernel.org/r/20200726001731.19540-1-rdunlap@infradead.org Signed-off-by: Juergen Gross --- arch/x86/include/asm/xen/hypercall.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/arch/x86/include/asm/xen/hypercall.h b/arch/x86/include/asm/xen/hypercall.h index ba4c1b15908b..454b20815f35 100644 --- a/arch/x86/include/asm/xen/hypercall.h +++ b/arch/x86/include/asm/xen/hypercall.h @@ -82,7 +82,7 @@ struct xen_dm_op_buf; * - clobber the rest * * The result certainly isn't pretty, and it really shows up cpp's - * weakness as as macro language. Sorry. (But let's just give thanks + * weakness as a macro language. Sorry. (But let's just give thanks * there aren't more than 5 arguments...) */ From 1951fa33ec259abdf3497bfee7b63e7ddbb1a394 Mon Sep 17 00:00:00 2001 From: Roger Pau Monne Date: Mon, 27 Jul 2020 11:13:39 +0200 Subject: [PATCH 06/19] xen/balloon: fix accounting in alloc_xenballooned_pages error path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit target_unpopulated is incremented with nr_pages at the start of the function, but the call to free_xenballooned_pages will only subtract pgno number of pages, and thus the rest need to be subtracted before returning or else accounting will be skewed. Signed-off-by: Roger Pau Monné Reviewed-by: Juergen Gross Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200727091342.52325-2-roger.pau@citrix.com Signed-off-by: Juergen Gross --- drivers/xen/balloon.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 77c57568e5d7..3cb10ed32557 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -630,6 +630,12 @@ int alloc_xenballooned_pages(int nr_pages, struct page **pages) out_undo: mutex_unlock(&balloon_mutex); free_xenballooned_pages(pgno, pages); + /* + * NB: free_xenballooned_pages will only subtract pgno pages, but since + * target_unpopulated is incremented with nr_pages at the start we need + * to remove the remaining ones also, or accounting will be screwed. + */ + balloon_stats.target_unpopulated -= nr_pages - pgno; return ret; } EXPORT_SYMBOL(alloc_xenballooned_pages); From 88a479ff6ef8af7f07e11593d58befc644244ff7 Mon Sep 17 00:00:00 2001 From: Roger Pau Monne Date: Mon, 27 Jul 2020 11:13:40 +0200 Subject: [PATCH 07/19] xen/balloon: make the balloon wait interruptible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit So it can be killed, or else processes can get hung indefinitely waiting for balloon pages. Signed-off-by: Roger Pau Monné Reviewed-by: Juergen Gross Cc: stable@vger.kernel.org Link: https://lore.kernel.org/r/20200727091342.52325-3-roger.pau@citrix.com Signed-off-by: Juergen Gross --- drivers/xen/balloon.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 3cb10ed32557..292413b27575 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -568,11 +568,13 @@ static int add_ballooned_pages(int nr_pages) if (xen_hotplug_unpopulated) { st = reserve_additional_memory(); if (st != BP_ECANCELED) { + int rc; + mutex_unlock(&balloon_mutex); - wait_event(balloon_wq, + rc = wait_event_interruptible(balloon_wq, !list_empty(&ballooned_pages)); mutex_lock(&balloon_mutex); - return 0; + return rc ? -ENOMEM : 0; } } From f5ec6723269d9652ce42927cc03485061d663b23 Mon Sep 17 00:00:00 2001 From: Roger Pau Monne Date: Mon, 27 Jul 2020 11:13:41 +0200 Subject: [PATCH 08/19] Revert "xen/balloon: Fix crash when ballooning on x86 32 bit PAE" MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit dfd74a1edfaba5864276a2859190a8d242d18952. This has been fixed by commit dca4436d1cf9e0d237c which added the out of bounds check to __add_memory, so that trying to add blocks past MAX_PHYSMEM_BITS will fail. Note the check in the Xen balloon driver was bogus anyway, as it checked the start address of the resource, but it should instead test the end address to assert the whole resource falls below MAX_PHYSMEM_BITS. Signed-off-by: Roger Pau Monné Reviewed-by: Juergen Gross Link: https://lore.kernel.org/r/20200727091342.52325-4-roger.pau@citrix.com Signed-off-by: Juergen Gross --- drivers/xen/balloon.c | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/drivers/xen/balloon.c b/drivers/xen/balloon.c index 292413b27575..b1d8b028bf80 100644 --- a/drivers/xen/balloon.c +++ b/drivers/xen/balloon.c @@ -266,20 +266,6 @@ static struct resource *additional_memory_resource(phys_addr_t size) return NULL; } -#ifdef CONFIG_SPARSEMEM - { - unsigned long limit = 1UL << (MAX_PHYSMEM_BITS - PAGE_SHIFT); - unsigned long pfn = res->start >> PAGE_SHIFT; - - if (pfn > limit) { - pr_err("New System RAM resource outside addressable RAM (%lu > %lu)\n", - pfn, limit); - release_memory_resource(res); - return NULL; - } - } -#endif - return res; } From 8b1e868f66076490189a36d984fcce286cdd6295 Mon Sep 17 00:00:00 2001 From: Boris Ostrovsky Date: Fri, 10 Jul 2020 15:34:17 -0700 Subject: [PATCH 09/19] swiotlb-xen: use vmalloc_to_page on vmalloc virt addresses xen_alloc_coherent_pages might return pages for which virt_to_phys and virt_to_page don't work, e.g. ioremap'ed pages. So in xen_swiotlb_free_coherent we can't assume that virt_to_page works. Instead add a is_vmalloc_addr check and use vmalloc_to_page on vmalloc virt addresses. This patch fixes the following crash at boot on RPi4 (the underlying issue is not RPi4 specific): https://marc.info/?l=xen-devel&m=158862573216800 Signed-off-by: Boris Ostrovsky Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-1-sstabellini@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/swiotlb-xen.c | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index b6d27762c6f8..5fbadd07819b 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -335,6 +335,7 @@ xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, int order = get_order(size); phys_addr_t phys; u64 dma_mask = DMA_BIT_MASK(32); + struct page *page; if (hwdev && hwdev->coherent_dma_mask) dma_mask = hwdev->coherent_dma_mask; @@ -346,9 +347,14 @@ xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, /* Convert the size to actually allocated. */ size = 1UL << (order + XEN_PAGE_SHIFT); + if (is_vmalloc_addr(vaddr)) + page = vmalloc_to_page(vaddr); + else + page = virt_to_page(vaddr); + if (!WARN_ON((dev_addr + size - 1 > dma_mask) || range_straddles_page_boundary(phys, size)) && - TestClearPageXenRemapped(virt_to_page(vaddr))) + TestClearPageXenRemapped(page)) xen_destroy_contiguous_region(phys, order); xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs); From ae4f0a17ee591c15efbee87746290d22aea81617 Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:18 -0700 Subject: [PATCH 10/19] swiotlb-xen: remove start_dma_addr It is not strictly needed. Call virt_to_phys on xen_io_tlb_start instead. It will be useful not to have a start_dma_addr around with the next patches. Note that virt_to_phys is not the same as xen_virt_to_bus but actually it is used to compared again __pa(xen_io_tlb_start) as passed to swiotlb_init_with_tbl, so virt_to_phys is actually what we want. Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-2-sstabellini@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/swiotlb-xen.c | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 5fbadd07819b..89a775948a02 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -52,8 +52,6 @@ static unsigned long xen_io_tlb_nslabs; * Quick lookup value of the bus address of the IOTLB. */ -static u64 start_dma_addr; - /* * Both of these functions should avoid XEN_PFN_PHYS because phys_addr_t * can be 32bit when dma_addr_t is 64bit leading to a loss in @@ -241,7 +239,6 @@ int __ref xen_swiotlb_init(int verbose, bool early) m_ret = XEN_SWIOTLB_EFIXUP; goto error; } - start_dma_addr = xen_virt_to_bus(xen_io_tlb_start); if (early) { if (swiotlb_init_with_tbl(xen_io_tlb_start, xen_io_tlb_nslabs, verbose)) @@ -392,8 +389,8 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, */ trace_swiotlb_bounced(dev, dev_addr, size, swiotlb_force); - map = swiotlb_tbl_map_single(dev, start_dma_addr, phys, - size, size, dir, attrs); + map = swiotlb_tbl_map_single(dev, virt_to_phys(xen_io_tlb_start), + phys, size, size, dir, attrs); if (map == (phys_addr_t)DMA_MAPPING_ERROR) return DMA_MAPPING_ERROR; From 2cf6a91347b94528c646cbdd9d295770ea7d447d Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:19 -0700 Subject: [PATCH 11/19] swiotlb-xen: add struct device * parameter to xen_phys_to_bus No functional changes. The parameter is unused in this patch but will be used by next patches. Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-3-sstabellini@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/swiotlb-xen.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 89a775948a02..dbe710a59bf2 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -57,7 +57,7 @@ static unsigned long xen_io_tlb_nslabs; * can be 32bit when dma_addr_t is 64bit leading to a loss in * information if the shift is done before casting to 64bit. */ -static inline dma_addr_t xen_phys_to_bus(phys_addr_t paddr) +static inline dma_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr) { unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr)); dma_addr_t dma = (dma_addr_t)bfn << XEN_PAGE_SHIFT; @@ -78,9 +78,9 @@ static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr) return paddr; } -static inline dma_addr_t xen_virt_to_bus(void *address) +static inline dma_addr_t xen_virt_to_bus(struct device *dev, void *address) { - return xen_phys_to_bus(virt_to_phys(address)); + return xen_phys_to_bus(dev, virt_to_phys(address)); } static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) @@ -309,7 +309,7 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, * Do not use virt_to_phys(ret) because on ARM it doesn't correspond * to *dma_handle. */ phys = *dma_handle; - dev_addr = xen_phys_to_bus(phys); + dev_addr = xen_phys_to_bus(hwdev, phys); if (((dev_addr + size - 1 <= dma_mask)) && !range_straddles_page_boundary(phys, size)) *dma_handle = dev_addr; @@ -370,7 +370,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, unsigned long attrs) { phys_addr_t map, phys = page_to_phys(page) + offset; - dma_addr_t dev_addr = xen_phys_to_bus(phys); + dma_addr_t dev_addr = xen_phys_to_bus(dev, phys); BUG_ON(dir == DMA_NONE); /* @@ -395,7 +395,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, return DMA_MAPPING_ERROR; phys = map; - dev_addr = xen_phys_to_bus(map); + dev_addr = xen_phys_to_bus(dev, map); /* * Ensure that the address returned is DMA'ble @@ -539,7 +539,7 @@ xen_swiotlb_sync_sg_for_device(struct device *dev, struct scatterlist *sgl, static int xen_swiotlb_dma_supported(struct device *hwdev, u64 mask) { - return xen_virt_to_bus(xen_io_tlb_end - 1) <= mask; + return xen_virt_to_bus(hwdev, xen_io_tlb_end - 1) <= mask; } const struct dma_map_ops xen_swiotlb_dma_ops = { From d900781a170da5ed0fc9840c260384bac0710c09 Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:20 -0700 Subject: [PATCH 12/19] swiotlb-xen: add struct device * parameter to xen_bus_to_phys No functional changes. The parameter is unused in this patch but will be used by next patches. Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-4-sstabellini@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/swiotlb-xen.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index dbe710a59bf2..a8e447137faf 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -67,7 +67,7 @@ static inline dma_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr) return dma; } -static inline phys_addr_t xen_bus_to_phys(dma_addr_t baddr) +static inline phys_addr_t xen_bus_to_phys(struct device *dev, dma_addr_t baddr) { unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr)); dma_addr_t dma = (dma_addr_t)xen_pfn << XEN_PAGE_SHIFT; @@ -339,7 +339,7 @@ xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, /* do not use virt_to_phys because on ARM it doesn't return you the * physical address */ - phys = xen_bus_to_phys(dev_addr); + phys = xen_bus_to_phys(hwdev, dev_addr); /* Convert the size to actually allocated. */ size = 1UL << (order + XEN_PAGE_SHIFT); @@ -423,7 +423,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { - phys_addr_t paddr = xen_bus_to_phys(dev_addr); + phys_addr_t paddr = xen_bus_to_phys(hwdev, dev_addr); BUG_ON(dir == DMA_NONE); @@ -439,7 +439,7 @@ static void xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { - phys_addr_t paddr = xen_bus_to_phys(dma_addr); + phys_addr_t paddr = xen_bus_to_phys(dev, dma_addr); if (!dev_is_dma_coherent(dev)) xen_dma_sync_for_cpu(dma_addr, paddr, size, dir); @@ -452,7 +452,7 @@ static void xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { - phys_addr_t paddr = xen_bus_to_phys(dma_addr); + phys_addr_t paddr = xen_bus_to_phys(dev, dma_addr); if (is_xen_swiotlb_buffer(dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); From 297f7f82636115d6db4211b8d81764223baf2908 Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:21 -0700 Subject: [PATCH 13/19] swiotlb-xen: add struct device * parameter to xen_dma_sync_for_cpu No functional changes. The parameter is unused in this patch but will be used by next patches. Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-5-sstabellini@kernel.org Signed-off-by: Juergen Gross --- arch/arm/xen/mm.c | 5 +++-- drivers/xen/swiotlb-xen.c | 4 ++-- include/xen/swiotlb-xen.h | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index d40e9e5fc52b..1a00e8003c64 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -71,8 +71,9 @@ static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op) * pfn_valid returns true the pages is local and we can use the native * dma-direct functions, otherwise we call the Xen specific version. */ -void xen_dma_sync_for_cpu(dma_addr_t handle, phys_addr_t paddr, size_t size, - enum dma_data_direction dir) +void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, + phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { if (pfn_valid(PFN_DOWN(handle))) arch_sync_dma_for_cpu(paddr, size, dir); diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index a8e447137faf..d04b7a15124f 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -428,7 +428,7 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, BUG_ON(dir == DMA_NONE); if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - xen_dma_sync_for_cpu(dev_addr, paddr, size, dir); + xen_dma_sync_for_cpu(hwdev, dev_addr, paddr, size, dir); /* NOTE: We use dev_addr here, not paddr! */ if (is_xen_swiotlb_buffer(dev_addr)) @@ -442,7 +442,7 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, phys_addr_t paddr = xen_bus_to_phys(dev, dma_addr); if (!dev_is_dma_coherent(dev)) - xen_dma_sync_for_cpu(dma_addr, paddr, size, dir); + xen_dma_sync_for_cpu(dev, dma_addr, paddr, size, dir); if (is_xen_swiotlb_buffer(dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU); diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h index ffc0d3902b71..f62d1854780b 100644 --- a/include/xen/swiotlb-xen.h +++ b/include/xen/swiotlb-xen.h @@ -4,8 +4,9 @@ #include -void xen_dma_sync_for_cpu(dma_addr_t handle, phys_addr_t paddr, size_t size, - enum dma_data_direction dir); +void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, + phys_addr_t paddr, size_t size, + enum dma_data_direction dir); void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size, enum dma_data_direction dir); From 995d3556694edd3c6dda7671b46ad4a6b3043f2f Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:22 -0700 Subject: [PATCH 14/19] swiotlb-xen: add struct device * parameter to xen_dma_sync_for_device No functional changes. The parameter is unused in this patch but will be used by next patches. Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-6-sstabellini@kernel.org Signed-off-by: Juergen Gross --- arch/arm/xen/mm.c | 5 +++-- drivers/xen/swiotlb-xen.c | 4 ++-- include/xen/swiotlb-xen.h | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index 1a00e8003c64..f2414ea40a79 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -81,8 +81,9 @@ void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); } -void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size, - enum dma_data_direction dir) +void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, + phys_addr_t paddr, size_t size, + enum dma_data_direction dir) { if (pfn_valid(PFN_DOWN(handle))) arch_sync_dma_for_device(paddr, size, dir); diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index d04b7a15124f..8a3a7bcc5ec0 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -408,7 +408,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, done: if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - xen_dma_sync_for_device(dev_addr, phys, size, dir); + xen_dma_sync_for_device(dev, dev_addr, phys, size, dir); return dev_addr; } @@ -458,7 +458,7 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); if (!dev_is_dma_coherent(dev)) - xen_dma_sync_for_device(dma_addr, paddr, size, dir); + xen_dma_sync_for_device(dev, dma_addr, paddr, size, dir); } /* diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h index f62d1854780b..6d235fe2b92d 100644 --- a/include/xen/swiotlb-xen.h +++ b/include/xen/swiotlb-xen.h @@ -7,8 +7,9 @@ void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, phys_addr_t paddr, size_t size, enum dma_data_direction dir); -void xen_dma_sync_for_device(dma_addr_t handle, phys_addr_t paddr, size_t size, - enum dma_data_direction dir); +void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, + phys_addr_t paddr, size_t size, + enum dma_data_direction dir); extern int xen_swiotlb_init(int verbose, bool early); extern const struct dma_map_ops xen_swiotlb_dma_ops; From 38ba51de46767cbcef04305c3e8a99f0965fc9df Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:23 -0700 Subject: [PATCH 15/19] swiotlb-xen: add struct device * parameter to is_xen_swiotlb_buffer No functional changes. The parameter is unused in this patch but will be used by next patches. Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-7-sstabellini@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/swiotlb-xen.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 8a3a7bcc5ec0..e2c35f45f91e 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -97,7 +97,7 @@ static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) return 0; } -static int is_xen_swiotlb_buffer(dma_addr_t dma_addr) +static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr) { unsigned long bfn = XEN_PFN_DOWN(dma_addr); unsigned long xen_pfn = bfn_to_local_pfn(bfn); @@ -431,7 +431,7 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, xen_dma_sync_for_cpu(hwdev, dev_addr, paddr, size, dir); /* NOTE: We use dev_addr here, not paddr! */ - if (is_xen_swiotlb_buffer(dev_addr)) + if (is_xen_swiotlb_buffer(hwdev, dev_addr)) swiotlb_tbl_unmap_single(hwdev, paddr, size, size, dir, attrs); } @@ -444,7 +444,7 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, if (!dev_is_dma_coherent(dev)) xen_dma_sync_for_cpu(dev, dma_addr, paddr, size, dir); - if (is_xen_swiotlb_buffer(dma_addr)) + if (is_xen_swiotlb_buffer(dev, dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU); } @@ -454,7 +454,7 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, { phys_addr_t paddr = xen_bus_to_phys(dev, dma_addr); - if (is_xen_swiotlb_buffer(dma_addr)) + if (is_xen_swiotlb_buffer(dev, dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); if (!dev_is_dma_coherent(dev)) From e9aab7e4ffbbf6eba8eaa66e351557c0c7562cff Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:24 -0700 Subject: [PATCH 16/19] swiotlb-xen: remove XEN_PFN_PHYS XEN_PFN_PHYS is only used in one place in swiotlb-xen making things more complex than need to be. Remove the definition of XEN_PFN_PHYS and open code the cast in the one place where it is needed. Signed-off-by: Stefano Stabellini Reviewed-by: Juergen Gross Link: https://lore.kernel.org/r/20200710223427.6897-8-sstabellini@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/swiotlb-xen.c | 7 +------ include/xen/page.h | 1 - 2 files changed, 1 insertion(+), 7 deletions(-) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index e2c35f45f91e..03d118b6c141 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -52,11 +52,6 @@ static unsigned long xen_io_tlb_nslabs; * Quick lookup value of the bus address of the IOTLB. */ -/* - * Both of these functions should avoid XEN_PFN_PHYS because phys_addr_t - * can be 32bit when dma_addr_t is 64bit leading to a loss in - * information if the shift is done before casting to 64bit. - */ static inline dma_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr) { unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr)); @@ -101,7 +96,7 @@ static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr) { unsigned long bfn = XEN_PFN_DOWN(dma_addr); unsigned long xen_pfn = bfn_to_local_pfn(bfn); - phys_addr_t paddr = XEN_PFN_PHYS(xen_pfn); + phys_addr_t paddr = (phys_addr_t)xen_pfn << XEN_PAGE_SHIFT; /* If the address is outside our domain, it CAN * have the same virtual address as another address diff --git a/include/xen/page.h b/include/xen/page.h index df6d6b6ec66e..285677b42943 100644 --- a/include/xen/page.h +++ b/include/xen/page.h @@ -24,7 +24,6 @@ #define XEN_PFN_DOWN(x) ((x) >> XEN_PAGE_SHIFT) #define XEN_PFN_UP(x) (((x) + XEN_PAGE_SIZE-1) >> XEN_PAGE_SHIFT) -#define XEN_PFN_PHYS(x) ((phys_addr_t)(x) << XEN_PAGE_SHIFT) #include From 91ffe4ad534ab2262cbcfb339c5ae190c7ac0e7c Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:25 -0700 Subject: [PATCH 17/19] swiotlb-xen: introduce phys_to_dma/dma_to_phys translations With some devices physical addresses are different than dma addresses. To be able to deal with these cases, we need to call phys_to_dma on physical addresses (including machine addresses in Xen terminology) before returning them from xen_swiotlb_alloc_coherent and xen_swiotlb_map_page. We also need to convert dma addresses back to physical addresses using dma_to_phys in xen_swiotlb_free_coherent and xen_swiotlb_unmap_page if we want to do any operations on them. Call dma_to_phys in is_xen_swiotlb_buffer. Introduce xen_phys_to_dma and call phys_to_dma in its implementation. Introduce xen_dma_to_phys and call dma_to_phys in its implementation. Call xen_phys_to_dma/xen_dma_to_phys instead of xen_phys_to_bus/xen_bus_to_phys through swiotlb-xen.c. Everything is taken care of by these changes except for xen_swiotlb_alloc_coherent and xen_swiotlb_free_coherent, which need a few explicit phys_to_dma/dma_to_phys calls. Signed-off-by: Stefano Stabellini Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Reviewed-by: Juergen Gross Link: https://lore.kernel.org/r/20200710223427.6897-9-sstabellini@kernel.org Signed-off-by: Juergen Gross --- drivers/xen/swiotlb-xen.c | 55 +++++++++++++++++++++++---------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index 03d118b6c141..a6a95358a8cb 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -52,30 +52,39 @@ static unsigned long xen_io_tlb_nslabs; * Quick lookup value of the bus address of the IOTLB. */ -static inline dma_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr) +static inline phys_addr_t xen_phys_to_bus(struct device *dev, phys_addr_t paddr) { unsigned long bfn = pfn_to_bfn(XEN_PFN_DOWN(paddr)); - dma_addr_t dma = (dma_addr_t)bfn << XEN_PAGE_SHIFT; + phys_addr_t baddr = (phys_addr_t)bfn << XEN_PAGE_SHIFT; - dma |= paddr & ~XEN_PAGE_MASK; - - return dma; + baddr |= paddr & ~XEN_PAGE_MASK; + return baddr; } -static inline phys_addr_t xen_bus_to_phys(struct device *dev, dma_addr_t baddr) +static inline dma_addr_t xen_phys_to_dma(struct device *dev, phys_addr_t paddr) +{ + return phys_to_dma(dev, xen_phys_to_bus(dev, paddr)); +} + +static inline phys_addr_t xen_bus_to_phys(struct device *dev, + phys_addr_t baddr) { unsigned long xen_pfn = bfn_to_pfn(XEN_PFN_DOWN(baddr)); - dma_addr_t dma = (dma_addr_t)xen_pfn << XEN_PAGE_SHIFT; - phys_addr_t paddr = dma; - - paddr |= baddr & ~XEN_PAGE_MASK; + phys_addr_t paddr = (xen_pfn << XEN_PAGE_SHIFT) | + (baddr & ~XEN_PAGE_MASK); return paddr; } +static inline phys_addr_t xen_dma_to_phys(struct device *dev, + dma_addr_t dma_addr) +{ + return xen_bus_to_phys(dev, dma_to_phys(dev, dma_addr)); +} + static inline dma_addr_t xen_virt_to_bus(struct device *dev, void *address) { - return xen_phys_to_bus(dev, virt_to_phys(address)); + return xen_phys_to_dma(dev, virt_to_phys(address)); } static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) @@ -94,7 +103,7 @@ static inline int range_straddles_page_boundary(phys_addr_t p, size_t size) static int is_xen_swiotlb_buffer(struct device *dev, dma_addr_t dma_addr) { - unsigned long bfn = XEN_PFN_DOWN(dma_addr); + unsigned long bfn = XEN_PFN_DOWN(dma_to_phys(dev, dma_addr)); unsigned long xen_pfn = bfn_to_local_pfn(bfn); phys_addr_t paddr = (phys_addr_t)xen_pfn << XEN_PAGE_SHIFT; @@ -299,12 +308,12 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, if (hwdev && hwdev->coherent_dma_mask) dma_mask = hwdev->coherent_dma_mask; - /* At this point dma_handle is the physical address, next we are + /* At this point dma_handle is the dma address, next we are * going to set it to the machine address. * Do not use virt_to_phys(ret) because on ARM it doesn't correspond * to *dma_handle. */ - phys = *dma_handle; - dev_addr = xen_phys_to_bus(hwdev, phys); + phys = dma_to_phys(hwdev, *dma_handle); + dev_addr = xen_phys_to_dma(hwdev, phys); if (((dev_addr + size - 1 <= dma_mask)) && !range_straddles_page_boundary(phys, size)) *dma_handle = dev_addr; @@ -314,6 +323,7 @@ xen_swiotlb_alloc_coherent(struct device *hwdev, size_t size, xen_free_coherent_pages(hwdev, size, ret, (dma_addr_t)phys, attrs); return NULL; } + *dma_handle = phys_to_dma(hwdev, *dma_handle); SetPageXenRemapped(virt_to_page(ret)); } memset(ret, 0, size); @@ -334,7 +344,7 @@ xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, /* do not use virt_to_phys because on ARM it doesn't return you the * physical address */ - phys = xen_bus_to_phys(hwdev, dev_addr); + phys = xen_dma_to_phys(hwdev, dev_addr); /* Convert the size to actually allocated. */ size = 1UL << (order + XEN_PAGE_SHIFT); @@ -349,7 +359,8 @@ xen_swiotlb_free_coherent(struct device *hwdev, size_t size, void *vaddr, TestClearPageXenRemapped(page)) xen_destroy_contiguous_region(phys, order); - xen_free_coherent_pages(hwdev, size, vaddr, (dma_addr_t)phys, attrs); + xen_free_coherent_pages(hwdev, size, vaddr, phys_to_dma(hwdev, phys), + attrs); } /* @@ -365,7 +376,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, unsigned long attrs) { phys_addr_t map, phys = page_to_phys(page) + offset; - dma_addr_t dev_addr = xen_phys_to_bus(dev, phys); + dma_addr_t dev_addr = xen_phys_to_dma(dev, phys); BUG_ON(dir == DMA_NONE); /* @@ -390,7 +401,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, return DMA_MAPPING_ERROR; phys = map; - dev_addr = xen_phys_to_bus(dev, map); + dev_addr = xen_phys_to_dma(dev, map); /* * Ensure that the address returned is DMA'ble @@ -418,7 +429,7 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, size_t size, enum dma_data_direction dir, unsigned long attrs) { - phys_addr_t paddr = xen_bus_to_phys(hwdev, dev_addr); + phys_addr_t paddr = xen_dma_to_phys(hwdev, dev_addr); BUG_ON(dir == DMA_NONE); @@ -434,7 +445,7 @@ static void xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { - phys_addr_t paddr = xen_bus_to_phys(dev, dma_addr); + phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr); if (!dev_is_dma_coherent(dev)) xen_dma_sync_for_cpu(dev, dma_addr, paddr, size, dir); @@ -447,7 +458,7 @@ static void xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, size_t size, enum dma_data_direction dir) { - phys_addr_t paddr = xen_bus_to_phys(dev, dma_addr); + phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr); if (is_xen_swiotlb_buffer(dev, dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); From 63f0620cc552c4cd5bb2747f77efce407487cb12 Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:26 -0700 Subject: [PATCH 18/19] xen/arm: introduce phys/dma translations in xen_dma_sync_for_* xen_dma_sync_for_cpu, xen_dma_sync_for_device, xen_arch_need_swiotlb are getting called passing dma addresses. On some platforms dma addresses could be different from physical addresses. Before doing any operations on these addresses we need to convert them back to physical addresses using dma_to_phys. Move the arch_sync_dma_for_cpu and arch_sync_dma_for_device calls from xen_dma_sync_for_cpu/device to swiotlb-xen.c, and add a call dma_to_phys to do address translations there. dma_cache_maint is fixed by the next patch. Signed-off-by: Stefano Stabellini Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Acked-by: Juergen Gross Link: https://lore.kernel.org/r/20200710223427.6897-10-sstabellini@kernel.org Signed-off-by: Juergen Gross --- arch/arm/xen/mm.c | 17 ++++++----------- drivers/xen/swiotlb-xen.c | 32 ++++++++++++++++++++++++-------- include/xen/swiotlb-xen.h | 6 ++---- 3 files changed, 32 insertions(+), 23 deletions(-) diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index f2414ea40a79..a8251a70f442 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -1,5 +1,6 @@ // SPDX-License-Identifier: GPL-2.0-only #include +#include #include #include #include @@ -72,22 +73,16 @@ static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op) * dma-direct functions, otherwise we call the Xen specific version. */ void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, - enum dma_data_direction dir) + size_t size, enum dma_data_direction dir) { - if (pfn_valid(PFN_DOWN(handle))) - arch_sync_dma_for_cpu(paddr, size, dir); - else if (dir != DMA_TO_DEVICE) + if (dir != DMA_TO_DEVICE) dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); } void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, - enum dma_data_direction dir) + size_t size, enum dma_data_direction dir) { - if (pfn_valid(PFN_DOWN(handle))) - arch_sync_dma_for_device(paddr, size, dir); - else if (dir == DMA_FROM_DEVICE) + if (dir == DMA_FROM_DEVICE) dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); else dma_cache_maint(handle, size, GNTTAB_CACHE_CLEAN); @@ -98,7 +93,7 @@ bool xen_arch_need_swiotlb(struct device *dev, dma_addr_t dev_addr) { unsigned int xen_pfn = XEN_PFN_DOWN(phys); - unsigned int bfn = XEN_PFN_DOWN(dev_addr); + unsigned int bfn = XEN_PFN_DOWN(dma_to_phys(dev, dev_addr)); /* * The swiotlb buffer should be used if diff --git a/drivers/xen/swiotlb-xen.c b/drivers/xen/swiotlb-xen.c index a6a95358a8cb..39a0f2e0847c 100644 --- a/drivers/xen/swiotlb-xen.c +++ b/drivers/xen/swiotlb-xen.c @@ -413,8 +413,12 @@ static dma_addr_t xen_swiotlb_map_page(struct device *dev, struct page *page, } done: - if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - xen_dma_sync_for_device(dev, dev_addr, phys, size, dir); + if (!dev_is_dma_coherent(dev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { + if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dev_addr)))) + arch_sync_dma_for_device(phys, size, dir); + else + xen_dma_sync_for_device(dev, dev_addr, size, dir); + } return dev_addr; } @@ -433,8 +437,12 @@ static void xen_swiotlb_unmap_page(struct device *hwdev, dma_addr_t dev_addr, BUG_ON(dir == DMA_NONE); - if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) - xen_dma_sync_for_cpu(hwdev, dev_addr, paddr, size, dir); + if (!dev_is_dma_coherent(hwdev) && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { + if (pfn_valid(PFN_DOWN(dma_to_phys(hwdev, dev_addr)))) + arch_sync_dma_for_cpu(paddr, size, dir); + else + xen_dma_sync_for_cpu(hwdev, dev_addr, size, dir); + } /* NOTE: We use dev_addr here, not paddr! */ if (is_xen_swiotlb_buffer(hwdev, dev_addr)) @@ -447,8 +455,12 @@ xen_swiotlb_sync_single_for_cpu(struct device *dev, dma_addr_t dma_addr, { phys_addr_t paddr = xen_dma_to_phys(dev, dma_addr); - if (!dev_is_dma_coherent(dev)) - xen_dma_sync_for_cpu(dev, dma_addr, paddr, size, dir); + if (!dev_is_dma_coherent(dev)) { + if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr)))) + arch_sync_dma_for_cpu(paddr, size, dir); + else + xen_dma_sync_for_cpu(dev, dma_addr, size, dir); + } if (is_xen_swiotlb_buffer(dev, dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_CPU); @@ -463,8 +475,12 @@ xen_swiotlb_sync_single_for_device(struct device *dev, dma_addr_t dma_addr, if (is_xen_swiotlb_buffer(dev, dma_addr)) swiotlb_tbl_sync_single(dev, paddr, size, dir, SYNC_FOR_DEVICE); - if (!dev_is_dma_coherent(dev)) - xen_dma_sync_for_device(dev, dma_addr, paddr, size, dir); + if (!dev_is_dma_coherent(dev)) { + if (pfn_valid(PFN_DOWN(dma_to_phys(dev, dma_addr)))) + arch_sync_dma_for_device(paddr, size, dir); + else + xen_dma_sync_for_device(dev, dma_addr, size, dir); + } } /* diff --git a/include/xen/swiotlb-xen.h b/include/xen/swiotlb-xen.h index 6d235fe2b92d..d5eaf9d682b8 100644 --- a/include/xen/swiotlb-xen.h +++ b/include/xen/swiotlb-xen.h @@ -5,11 +5,9 @@ #include void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, - enum dma_data_direction dir); + size_t size, enum dma_data_direction dir); void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, - phys_addr_t paddr, size_t size, - enum dma_data_direction dir); + size_t size, enum dma_data_direction dir); extern int xen_swiotlb_init(int verbose, bool early); extern const struct dma_map_ops xen_swiotlb_dma_ops; From d7b461caa6cc64dd190577b46b0ec892a8d5e7c0 Mon Sep 17 00:00:00 2001 From: Stefano Stabellini Date: Fri, 10 Jul 2020 15:34:27 -0700 Subject: [PATCH 19/19] xen/arm: call dma_to_phys on the dma_addr_t parameter of dma_cache_maint dma_cache_maint is getting called passing a dma address which could be different from a physical address. Add a struct device* parameter to dma_cache_maint. Translate the dma_addr_t parameter of dma_cache_maint by calling dma_to_phys. Do it for the first page and all the following pages, in case of multipage handling. Signed-off-by: Stefano Stabellini Reviewed-by: Boris Ostrovsky Tested-by: Corey Minyard Tested-by: Roman Shaposhnik Link: https://lore.kernel.org/r/20200710223427.6897-11-sstabellini@kernel.org Signed-off-by: Juergen Gross --- arch/arm/xen/mm.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/arch/arm/xen/mm.c b/arch/arm/xen/mm.c index a8251a70f442..396797ffe2b1 100644 --- a/arch/arm/xen/mm.c +++ b/arch/arm/xen/mm.c @@ -43,15 +43,18 @@ unsigned long xen_get_swiotlb_free_pages(unsigned int order) static bool hypercall_cflush = false; /* buffers in highmem or foreign pages cannot cross page boundaries */ -static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op) +static void dma_cache_maint(struct device *dev, dma_addr_t handle, + size_t size, u32 op) { struct gnttab_cache_flush cflush; - cflush.a.dev_bus_addr = handle & XEN_PAGE_MASK; cflush.offset = xen_offset_in_page(handle); cflush.op = op; + handle &= XEN_PAGE_MASK; do { + cflush.a.dev_bus_addr = dma_to_phys(dev, handle); + if (size + cflush.offset > XEN_PAGE_SIZE) cflush.length = XEN_PAGE_SIZE - cflush.offset; else @@ -60,7 +63,7 @@ static void dma_cache_maint(dma_addr_t handle, size_t size, u32 op) HYPERVISOR_grant_table_op(GNTTABOP_cache_flush, &cflush, 1); cflush.offset = 0; - cflush.a.dev_bus_addr += cflush.length; + handle += cflush.length; size -= cflush.length; } while (size); } @@ -76,16 +79,16 @@ void xen_dma_sync_for_cpu(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { if (dir != DMA_TO_DEVICE) - dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); + dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); } void xen_dma_sync_for_device(struct device *dev, dma_addr_t handle, size_t size, enum dma_data_direction dir) { if (dir == DMA_FROM_DEVICE) - dma_cache_maint(handle, size, GNTTAB_CACHE_INVAL); + dma_cache_maint(dev, handle, size, GNTTAB_CACHE_INVAL); else - dma_cache_maint(handle, size, GNTTAB_CACHE_CLEAN); + dma_cache_maint(dev, handle, size, GNTTAB_CACHE_CLEAN); } bool xen_arch_need_swiotlb(struct device *dev,