[dma] Provide dma_umalloc() for allocating large DMA-coherent buffers

Some devices (e.g. xHCI USB host controllers) may require the use of
large areas of host memory for private use by the device.  These
allocations cannot be satisfied from iPXE's limited heap space, and so
are currently allocated using umalloc() which will allocate external
system memory (and alter the system memory map as needed).

Provide dma_umalloc() to provide such allocations as part of the DMA
API, since there is otherwise no way to guarantee that the allocated
regions are usable for coherent DMA.

Signed-off-by: Michael Brown <mcb30@ipxe.org>

3 files changed, 166 insertions, 0 deletions

diff --git a/src/core/dma.c b/src/core/dma.c
index e5fa3f323..5d6868216 100644
--- a/src/core/dma.c
+++ b/src/core/dma.c
@@ -44,6 +44,8 @@ PROVIDE_DMAAPI_INLINE ( flat, dma_map );
 PROVIDE_DMAAPI_INLINE ( flat, dma_unmap );
 PROVIDE_DMAAPI_INLINE ( flat, dma_alloc );
 PROVIDE_DMAAPI_INLINE ( flat, dma_free );
+PROVIDE_DMAAPI_INLINE ( flat, dma_umalloc );
+PROVIDE_DMAAPI_INLINE ( flat, dma_ufree );
 PROVIDE_DMAAPI_INLINE ( flat, dma_set_mask );
 PROVIDE_DMAAPI_INLINE ( flat, dma_phys );
 
@@ -120,6 +122,41 @@ static void dma_op_free ( struct dma_mapping *map, void *addr, size_t len ) {
 }
 
 /**
+ * Allocate and map DMA-coherent buffer from external (user) memory
+ *
+ * @v dma		DMA device
+ * @v map		DMA mapping to fill in
+ * @v len		Length of buffer
+ * @v align		Physical alignment
+ * @ret addr		Buffer address, or NULL on error
+ */
+static userptr_t dma_op_umalloc ( struct dma_device *dma,
+				  struct dma_mapping *map,
+				  size_t len, size_t align ) {
+	struct dma_operations *op = dma->op;
+
+	if ( ! op )
+		return UNULL;
+	return op->umalloc ( dma, map, len, align );
+}
+
+/**
+ * Unmap and free DMA-coherent buffer from external (user) memory
+ *
+ * @v map		DMA mapping
+ * @v addr		Buffer address
+ * @v len		Length of buffer
+ */
+static void dma_op_ufree ( struct dma_mapping *map, userptr_t addr,
+			   size_t len ) {
+	struct dma_device *dma = map->dma;
+
+	assert ( dma != NULL );
+	assert ( dma->op != NULL );
+	dma->op->ufree ( dma, map, addr, len );
+}
+
+/**
  * Set addressable space mask
  *
  * @v dma		DMA device
@@ -136,5 +173,7 @@ PROVIDE_DMAAPI ( op, dma_map, dma_op_map );
 PROVIDE_DMAAPI ( op, dma_unmap, dma_op_unmap );
 PROVIDE_DMAAPI ( op, dma_alloc, dma_op_alloc );
 PROVIDE_DMAAPI ( op, dma_free, dma_op_free );
+PROVIDE_DMAAPI ( op, dma_umalloc, dma_op_umalloc );
+PROVIDE_DMAAPI ( op, dma_ufree, dma_op_ufree );
 PROVIDE_DMAAPI ( op, dma_set_mask, dma_op_set_mask );
 PROVIDE_DMAAPI_INLINE ( op, dma_phys );
diff --git a/src/include/ipxe/dma.h b/src/include/ipxe/dma.h
index b3fa24e47..385e4baf7 100644
--- a/src/include/ipxe/dma.h
+++ b/src/include/ipxe/dma.h
@@ -13,6 +13,7 @@ FILE_LICENCE ( GPL2_OR_LATER_OR_UBDL );
 #include <ipxe/api.h>
 #include <ipxe/io.h>
 #include <ipxe/malloc.h>
+#include <ipxe/umalloc.h>
 #include <config/ioapi.h>
 
 #ifdef DMAAPI_OP
@@ -97,6 +98,28 @@ struct dma_operations {
 	void ( * free ) ( struct dma_device *dma, struct dma_mapping *map,
 			  void *addr, size_t len );
 	/**
+	 * Allocate and map DMA-coherent buffer from external (user) memory
+	 *
+	 * @v dma		DMA device
+	 * @v map		DMA mapping to fill in
+	 * @v len		Length of buffer
+	 * @v align		Physical alignment
+	 * @ret addr		Buffer address, or NULL on error
+	 */
+	userptr_t ( * umalloc ) ( struct dma_device *dma,
+				  struct dma_mapping *map,
+				  size_t len, size_t align );
+	/**
+	 * Unmap and free DMA-coherent buffer from external (user) memory
+	 *
+	 * @v dma		DMA device
+	 * @v map		DMA mapping
+	 * @v addr		Buffer address
+	 * @v len		Length of buffer
+	 */
+	void ( * ufree ) ( struct dma_device *dma, struct dma_mapping *map,
+			   userptr_t addr, size_t len );
+	/**
 	 * Set addressable space mask
 	 *
 	 * @v dma		DMA device
@@ -234,6 +257,55 @@ DMAAPI_INLINE ( flat, dma_free ) ( struct dma_mapping *map,
 }
 
 /**
+ * Allocate and map DMA-coherent buffer from external (user) memory
+ *
+ * @v dma		DMA device
+ * @v map		DMA mapping to fill in
+ * @v len		Length of buffer
+ * @v align		Physical alignment
+ * @ret addr		Buffer address, or NULL on error
+ */
+static inline __always_inline userptr_t
+DMAAPI_INLINE ( flat, dma_umalloc ) ( struct dma_device *dma,
+				      struct dma_mapping *map,
+				      size_t len, size_t align __unused ) {
+	userptr_t addr;
+
+	/* Allocate buffer */
+	addr = umalloc ( len );
+
+	/* Increment mapping count (for debugging) */
+	if ( DBG_LOG && addr ) {
+		map->dma = dma;
+		dma->mapped++;
+	}
+
+	return addr;
+}
+
+/**
+ * Unmap and free DMA-coherent buffer from external (user) memory
+ *
+ * @v map		DMA mapping
+ * @v addr		Buffer address
+ * @v len		Length of buffer
+ */
+static inline __always_inline void
+DMAAPI_INLINE ( flat, dma_ufree ) ( struct dma_mapping *map,
+				    userptr_t addr, size_t len __unused ) {
+
+	/* Free buffer */
+	ufree ( addr );
+
+	/* Decrement mapping count (for debugging) */
+	if ( DBG_LOG ) {
+		assert ( map->dma != NULL );
+		map->dma->mapped--;
+		map->dma = NULL;
+	}
+}
+
+/**
  * Set addressable space mask
  *
  * @v dma		DMA device
@@ -317,6 +389,27 @@ void * dma_alloc ( struct dma_device *dma, struct dma_mapping *map,
 void dma_free ( struct dma_mapping *map, void *addr, size_t len );
 
 /**
+ * Allocate and map DMA-coherent buffer from external (user) memory
+ *
+ * @v dma		DMA device
+ * @v map		DMA mapping to fill in
+ * @v len		Length of buffer
+ * @v align		Physical alignment
+ * @ret addr		Buffer address, or NULL on error
+ */
+userptr_t dma_umalloc ( struct dma_device *dma, struct dma_mapping *map,
+			size_t len, size_t align );
+
+/**
+ * Unmap and free DMA-coherent buffer from external (user) memory
+ *
+ * @v map		DMA mapping
+ * @v addr		Buffer address
+ * @v len		Length of buffer
+ */
+void dma_ufree ( struct dma_mapping *map, userptr_t addr, size_t len );
+
+/**
  * Set addressable space mask
  *
  * @v dma		DMA device
diff --git a/src/interface/efi/efi_pci.c b/src/interface/efi/efi_pci.c
index 9f6bf952b..6b32fd6a9 100644
--- a/src/interface/efi/efi_pci.c
+++ b/src/interface/efi/efi_pci.c
@@ -505,6 +505,38 @@ static void efipci_dma_free ( struct dma_device *dma, struct dma_mapping *map,
 }
 
 /**
+ * Allocate and map DMA-coherent buffer from external (user) memory
+ *
+ * @v dma		DMA device
+ * @v map		DMA mapping to fill in
+ * @v len		Length of buffer
+ * @v align		Physical alignment
+ * @ret addr		Buffer address, or NULL on error
+ */
+static userptr_t efipci_dma_umalloc ( struct dma_device *dma,
+				      struct dma_mapping *map,
+				      size_t len, size_t align ) {
+	void *addr;
+
+	addr = efipci_dma_alloc ( dma, map, len, align );
+	return virt_to_user ( addr );
+}
+
+/**
+ * Unmap and free DMA-coherent buffer from external (user) memory
+ *
+ * @v dma		DMA device
+ * @v map		DMA mapping
+ * @v addr		Buffer address
+ * @v len		Length of buffer
+ */
+static void efipci_dma_ufree ( struct dma_device *dma, struct dma_mapping *map,
+			       userptr_t addr, size_t len ) {
+
+	efipci_dma_free ( dma, map, user_to_virt ( addr, 0 ), len );
+}
+
+/**
  * Set addressable space mask
  *
  * @v dma		DMA device
@@ -542,6 +574,8 @@ static struct dma_operations efipci_dma_operations = {
 	.unmap = efipci_dma_unmap,
 	.alloc = efipci_dma_alloc,
 	.free = efipci_dma_free,
+	.umalloc = efipci_dma_umalloc,
+	.ufree = efipci_dma_ufree,
 	.set_mask = efipci_dma_set_mask,
 };