diff --git a/docs/reference/kernel-abi/index.rst b/docs/reference/kernel-abi/index.rst index 25045cc5..0a610c5a 100644 --- a/docs/reference/kernel-abi/index.rst +++ b/docs/reference/kernel-abi/index.rst @@ -426,12 +426,14 @@ All transfers are synchronous and block until the transfer completes or times ou **10 seconds**; after expiry the call returns ``-ETIME``. Partial transfers are possible; the return value is the number of bytes transferred, and the file position is advanced accordingly. -The userspace buffer address and ``count`` must be page-aligned: the address -must be 4 KiB-aligned and ``count`` must be a non-zero multiple of 4 KiB. The -transfer is backed by 4 KiB base pages, one descriptor per page. Transparent -hugepages are not accepted, so callers using anonymous mappings should apply -``MADV_NOHUGEPAGE`` before faulting pages when they need deterministic -base-page transfers. +The buffer offset must be aligned to the buffer's page granule. The transfer +length is an exact byte count: it must be non-zero and may end within the final +page. The transfer is backed by 4 KiB base pages, with the driver clipping the +final descriptor when the requested length is not page-multiple. +Non-page-multiple transfers are supported for exact byte-stream use cases such +as PDI programming, but they are not the speed-optimised path. Performance +critical application buffers should pad transfer lengths to full 4 KiB page +multiples whenever possible. Multiple fds can be obtained for the same qpair via multiple ``QPAIR_GET_FD`` calls, including from different processes. Concurrent ``read()``/``write()`` calls on the same qpair (from any @@ -451,7 +453,7 @@ The following errno values can be returned by ``read()`` and ``write()`` on the * - ``-ENODEV`` - Device shutting down, or the required direction is not enabled for this qpair * - ``-EINVAL`` - - Zero-length, unaligned, or non-page-multiple transfer + - Zero-length, unaligned, or out-of-range transfer * - ``-ENOMEM`` - SGL allocation failure * - ``-EFAULT`` @@ -565,10 +567,12 @@ is returned in the struct and is used for all subsequent operations on this queu __u32 size; /* [in/out] ABI version */ __u32 mode; /* [in] Queue mode: 0=MM (Memory Mapped), 1=ST (Streaming, not yet supported) */ __u32 dir_mask; /* [in] Direction bitmask (see below) */ + __u32 mm_channel; /* [in] AXI-MM/NoC channel selection: 0=auto, 1=channel 0, 2=channel 1 */ __u32 h2c_ring_sz; /* [in] H2C descriptor ring CSR table index: 0–15 */ __u32 c2h_ring_sz; /* [in] C2H descriptor ring CSR table index: 0–15 */ __u32 cmpt_ring_sz; /* [in] Completion ring CSR table index: 0–15 */ __u32 qid; /* [out] Kernel-assigned queue pair ID */ + __u32 aperture_size; /* [in] 0=linear MM addressing, non-zero=keyhole aperture size */ }; Direction bitmask bits: @@ -593,8 +597,14 @@ Ring size fields are QDMA Control and Status Register (CSR) table indices (0–1 descriptor counts. Index 0 maps to approximately 2049 descriptors; index 15 to approximately 16385. The caller does not control the actual descriptor count directly. -**Direction:** ``_IOWR`` — userspace writes ``mode``, ``dir_mask``, and ring size indices; the -kernel writes back ``qid``. +``aperture_size`` controls libqdma keyhole mode for memory-mapped queues. A value of ``0`` +keeps endpoint addressing linear and is the normal setting for DDR/HBM application buffers. +A non-zero power-of-two value enables keyhole mode: endpoint addresses wrap within that byte +aperture as the transfer advances. Keyhole queues are intended for special endpoints such as +the PDI design-writer ingress path; ordinary application queues should leave this field ``0``. + +**Direction:** ``_IOWR`` — userspace writes ``mode``, ``dir_mask``, ``mm_channel``, ring size +indices, and optionally ``aperture_size``; the kernel writes back ``qid``. **Preconditions:** @@ -602,7 +612,9 @@ kernel writes back ``qid``. - ``dir_mask`` must be non-zero and contain only bits ``[0, 1]``; bit 2 (CMPT) is not yet supported - ``mode`` must be 0 (MM); streaming mode (1) is not yet supported +- ``mm_channel`` must be 0 (auto), 1 (channel 0), or 2 (channel 1) - All ring size indices must be in ``[0, 15]`` +- ``aperture_size`` must be 0 (linear addressing) or a power-of-two keyhole aperture size - At most 256 concurrent queue pairs per device. The actual ceiling is lower in practice and depends on how many queues libqdma's resource manager makes available to the calling process (the 256-slot pool is shared across all PCI functions of the device). @@ -728,8 +740,8 @@ the new fd as the ``ioctl()`` return value (not as a struct field). - ``qpair_count`` must not exceed ``SLASH_QDMA_FD_MAX_QPAIRS`` - ``flags & ~O_CLOEXEC == 0`` (any other bits cause ``-EINVAL``) - The queue pairs should be in the started state for I/O to work -- Each bound qpair keeps the per-qpair configuration (``mm_channel``, ring sizes, directions) it was - given at ``QPAIR_ADD`` time, so the two channels can be configured independently +- Each bound qpair keeps the per-qpair configuration (``mm_channel``, ``aperture_size``, ring sizes, + directions) it was given at ``QPAIR_ADD`` time, so the two channels can be configured independently **Postconditions:** @@ -868,8 +880,9 @@ across all sub-transfers is returned as the ``ioctl()`` return value (not as a s - each sub-transfer's ``qpair_index`` must be ``< `` the number of qpairs the fd owns - each ``direction`` must be 1 (H2C) or 2 (C2H) and must be enabled on the selected queue pair - each ``buf_fd`` must be a buffer fd (from ``BUF_CREATE``) bound to the same device as this qpair fd -- each ``buf_offset`` and ``length`` must be aligned to the buffer's page granule, ``length`` non-zero - and ``<= UINT_MAX``, and ``buf_offset + length`` must not exceed the buffer length +- each ``buf_offset`` must be aligned to the buffer's page granule; ``length`` is an exact byte count, + must be non-zero and ``<= UINT_MAX``, may end within the final page, and ``buf_offset + length`` + must not exceed the buffer length **Return values:** diff --git a/driver/libslash/include/slash/uapi/slash_interface.h b/driver/libslash/include/slash/uapi/slash_interface.h index 1b1d85cb..501c8623 100644 --- a/driver/libslash/include/slash/uapi/slash_interface.h +++ b/driver/libslash/include/slash/uapi/slash_interface.h @@ -182,6 +182,11 @@ enum slash_qdma_mm_channel { * 0–15), not byte or descriptor counts. Each index selects a * pre-configured descriptor-ring depth from the global CSR ring-size * table (e.g. index 0 → 2049 descriptors, index 15 → 16385). + * + * \@aperture_size controls libqdma keyhole mode for memory-mapped queues: + * 0 keeps endpoint addressing linear; a non-zero power-of-two value wraps + * endpoint addresses within that byte aperture. Ordinary DDR/HBM queues + * should leave this 0. */ struct slash_qdma_qpair_add { __u32 size; /**< Struct size for ABI versioning. */ @@ -197,6 +202,9 @@ struct slash_qdma_qpair_add { /* Kernel to userspace */ __u32 qid; /**< [out] Kernel-assigned queue pair ID. */ + + /* Userspace to kernel; appended for ABI compatibility. */ + __u32 aperture_size; /**< [in] 0 = linear MM addressing, non-zero = keyhole aperture. */ }; /** @@ -325,10 +333,11 @@ struct slash_qdma_buf_create { * Moves \@length bytes between the kernel buffer named by \@buf_fd at * \@buf_offset and the device endpoint address \@dev_addr, on the queue pair * selected by \@qpair_index (an index into the fd's bound qpairs). - * \@buf_offset and \@length must be aligned to the buffer's 4 KiB page granule, - * and \@buf_offset + \@length must not exceed the buffer length. \@direction - * must be one of enum slash_qdma_transfer_dir and must be enabled on the - * selected queue pair. + * \@buf_offset must be aligned to the buffer's 4 KiB page granule. \@length + * is an exact, non-zero byte count that may end within the final page, and + * \@buf_offset + \@length must not exceed the buffer length. \@direction must + * be one of enum slash_qdma_transfer_dir and must be enabled on the selected + * queue pair. */ struct slash_qdma_subxfer { __u32 qpair_index; /**< [in] Index into the fd's bound qpairs. */ diff --git a/driver/libslash/src/qdma.c b/driver/libslash/src/qdma.c index efe8c3d3..5a0aa36e 100644 --- a/driver/libslash/src/qdma.c +++ b/driver/libslash/src/qdma.c @@ -323,6 +323,7 @@ int slash_qdma_qpair_add(struct slash_qdma *qdma, tmp.h2c_ring_sz = req->h2c_ring_sz; tmp.c2h_ring_sz = req->c2h_ring_sz; tmp.cmpt_ring_sz = req->cmpt_ring_sz; + tmp.aperture_size = req->aperture_size; ret = ioctl(qdma->fd, SLASH_QDMA_IOCTL_QPAIR_ADD, &tmp); if (ret < 0) { diff --git a/driver/libslash/tests/qdma_test.cpp b/driver/libslash/tests/qdma_test.cpp index 9519302e..23a70f7e 100644 --- a/driver/libslash/tests/qdma_test.cpp +++ b/driver/libslash/tests/qdma_test.cpp @@ -269,6 +269,65 @@ TEST_P(ParametrizedQdmaTest, BufferCreateTransfer) { EXPECT_EQ(slash_qdma_qpair_del(qdma_, qid), 0); } +TEST_P(ParametrizedQdmaTest, QpairAddAcceptsKeyholeAperture) { + struct slash_qdma_qpair_add req{}; + req.mode = 0; /* QDMA_Q_MODE_MM */ + req.dir_mask = 0x1; /* H2C */ + req.aperture_size = 4096; + + ASSERT_EQ(slash_qdma_qpair_add(qdma_, &req), 0); + uint32_t qid = req.qid; + + EXPECT_EQ(slash_qdma_qpair_start(qdma_, qid), 0); + EXPECT_EQ(slash_qdma_qpair_stop(qdma_, qid), 0); + EXPECT_EQ(slash_qdma_qpair_del(qdma_, qid), 0); +} + +TEST_P(ParametrizedQdmaTest, PartialLengthTransfer) { + static constexpr size_t BUFFER_SIZE = 4096; + static constexpr size_t XFER_SIZE = 4096 - 17; + + struct slash_qdma_qpair_add req{}; + req.mode = 0; /* QDMA_Q_MODE_MM */ + req.dir_mask = 0x3; /* H2C | C2H */ + + ASSERT_EQ(slash_qdma_qpair_add(qdma_, &req), 0); + uint32_t qid = req.qid; + ASSERT_EQ(slash_qdma_qpair_start(qdma_, qid), 0); + + int queue_fd = slash_qdma_qpair_get_fd(qdma_, qid, 0); + ASSERT_GE(queue_fd, 0); + + struct slash_qdma_buffer src_buf{}; + struct slash_qdma_buffer dst_buf{}; + ASSERT_EQ(slash_qdma_buffer_create(qdma_, BUFFER_SIZE, &src_buf), 0); + ASSERT_EQ(slash_qdma_buffer_create(qdma_, BUFFER_SIZE, &dst_buf), 0); + auto *src = static_cast(src_buf.addr); + auto *dst = static_cast(dst_buf.addr); + for (size_t i = 0; i < XFER_SIZE; ++i) { + src[i] = static_cast((i * 3 + 5) & 0xFF); + } + std::memset(dst, 0, BUFFER_SIZE); + + ssize_t written = slash_qdma_qpair_transfer(queue_fd, src_buf.fd, 0, + DDR_BASE_ADDRESS, XFER_SIZE, + SLASH_QDMA_XFER_H2C); + EXPECT_EQ(written, static_cast(XFER_SIZE)); + + ssize_t read_bytes = slash_qdma_qpair_transfer(queue_fd, dst_buf.fd, 0, + DDR_BASE_ADDRESS, XFER_SIZE, + SLASH_QDMA_XFER_C2H); + EXPECT_EQ(read_bytes, static_cast(XFER_SIZE)); + EXPECT_EQ(std::memcmp(src, dst, XFER_SIZE), 0); + + EXPECT_EQ(slash_qdma_buffer_destroy(&src_buf), 0); + EXPECT_EQ(slash_qdma_buffer_destroy(&dst_buf), 0); + + EXPECT_EQ(close(queue_fd), 0); + EXPECT_EQ(slash_qdma_qpair_stop(qdma_, qid), 0); + EXPECT_EQ(slash_qdma_qpair_del(qdma_, qid), 0); +} + TEST_P(ParametrizedQdmaTest, MultiQpairBatchTransfer) { // Two 4 KiB halves transferred concurrently across two queue pairs bound to // a single fd, exercising the get-fd-multi + batch transfer API. diff --git a/driver/slash_qdma.c b/driver/slash_qdma.c index aa4a8f47..d6c036ff 100644 --- a/driver/slash_qdma.c +++ b/driver/slash_qdma.c @@ -1832,6 +1832,8 @@ static void slash_qdma_ioctl_info(struct miscdevice *misc, * (completion queues are not yet supported). * - @mode must be MM; streaming mode (ST) is not yet supported. * - Ring size indices must be in [0, 15] (CSR table range). + * - @aperture_size must be zero (linear addressing) or a power-of-two + * libqdma keyhole aperture. * * On success, the kernel-assigned @qid is written back to userspace. * @@ -1895,6 +1897,10 @@ static int slash_qdma_ioctl_qpair_add_w(struct miscdevice *misc, req.mm_channel != SLASH_QDMA_MM_CHANNEL_1) return -EINVAL; + if (req.aperture_size != 0 && + (req.aperture_size & (req.aperture_size - 1)) != 0) + return -EINVAL; + mutex_lock(&qdma_dev->lock); if (qdma_dev->hw_shutdown || !qdma_dev->have_qdma_handle) { mutex_unlock(&qdma_dev->lock); @@ -2033,9 +2039,9 @@ static int slash_qdma_ioctl_qpair_add(struct miscdevice *misc, * (required for poll-mode operation per the reference driver). * - qconf.cmpl_stat_en = 1: enable completion status generation * (required for poll-mode operation per the reference driver). - * - qconf.aperture_size = 0: disables libqdma keyhole mode so MM - * transfers advance linearly through endpoint memory. Non-zero - * values are keyhole apertures and wrap addresses within that window. + * - qconf.aperture_size: zero disables libqdma keyhole mode so MM + * transfers advance linearly through endpoint memory. Non-zero values + * enable keyhole mode and wrap addresses within that byte aperture. * - qconf.desc_rng_sz_idx: CSR table index (0-15) selecting the * descriptor ring depth. Not a raw descriptor count — the actual * count is looked up from the global CSR ring-size table. @@ -2079,7 +2085,7 @@ static int slash_qdma_ioctl_qpair_add_q(struct miscdevice *misc, qconf.cmpl_status_pend_chk = 1; /* Check pending completions (poll-mode req) */ qconf.cmpl_stat_en = 1; /* Enable completion status generation */ - qconf.aperture_size = 0; /* Linear MM addressing; non-zero enables keyhole mode */ + qconf.aperture_size = req->aperture_size; /* 0 = linear MM; non-zero = keyhole aperture */ /* * CPM5 exposes two MM channels. The per-queue mm_channel selection * (validated in slash_qdma_ioctl_qpair_add_w) chooses the channel: AUTO @@ -2124,9 +2130,9 @@ static int slash_qdma_ioctl_qpair_add_q(struct miscdevice *misc, } SLASH_QDMA_OP_DEV_LOG(&qdma_dev->pdev->dev, - "queue add qid=%u type=%u mode=%u mm_channel=%u (req=%u)\n", + "queue add qid=%u type=%u mode=%u mm_channel=%u (req=%u) aperture_size=%u\n", req->qid, qtype, req->mode, qconf.mm_channel, - req->mm_channel); + req->mm_channel, qconf.aperture_size); err = qdma_queue_add(qdma_dev->qdma_handle, &qconf, &qhndl, errbuf, sizeof(errbuf)); if (err) { @@ -2603,49 +2609,52 @@ static int slash_qdma_buf_alloc(struct slash_qdma_buf *buf) } /** - * slash_qdma_buf_sync_for_device() - Hand a transfer slice to the device. - * @buf: Buffer being transferred. - * @start_entry: First page index of the slice. - * @n_entries: Number of pages in the slice. - * @dir: DMA direction (DMA_TO_DEVICE for H2C, DMA_FROM_DEVICE for C2H). + * slash_qdma_buf_sync_sgl_for_device() - Hand a transfer slice to the device. + * @buf: Buffer being transferred. + * @sgl: Per-transfer SGL slice. + * @n_entries: Number of SGL entries in @sgl. + * @dir: DMA direction (DMA_TO_DEVICE for H2C, DMA_FROM_DEVICE for C2H). * * Synchronises CPU-written data out to the device (and/or invalidates CPU - * caches) for exactly the pages a sub-transfer touches. On cache-coherent - * hosts these are no-ops; on others they bound coherency to the transfer. + * caches) for exactly the SGL spans a sub-transfer touches. On + * cache-coherent hosts these are no-ops; on others they bound coherency to + * the transfer. */ -static void slash_qdma_buf_sync_for_device(struct slash_qdma_buf *buf, - u64 start_entry, u64 n_entries, - enum dma_data_direction dir) +static void slash_qdma_buf_sync_sgl_for_device(struct slash_qdma_buf *buf, + const struct qdma_sw_sg *sgl, + u64 n_entries, + enum dma_data_direction dir) { struct device *dev = &buf->qdma_dev->pdev->dev; u64 i; for (i = 0; i < n_entries; i++) { - struct qdma_sw_sg *sg = &buf->sgl[start_entry + i]; + const struct qdma_sw_sg *sg = &sgl[i]; dma_sync_single_for_device(dev, sg->dma_addr, sg->len, dir); } } /** - * slash_qdma_buf_sync_for_cpu() - Reclaim a transfer slice for the CPU. - * @buf: Buffer being transferred. - * @start_entry: First page index of the slice. - * @n_entries: Number of pages in the slice. - * @dir: DMA direction (DMA_FROM_DEVICE for a completed C2H read). + * slash_qdma_buf_sync_sgl_for_cpu() - Reclaim a transfer slice for the CPU. + * @buf: Buffer being transferred. + * @sgl: Per-transfer SGL slice. + * @n_entries: Number of SGL entries in @sgl. + * @dir: DMA direction (DMA_FROM_DEVICE for a completed C2H read). * * Makes device-written data visible to the CPU for exactly the pages a C2H * sub-transfer touched. Called after the transfer completes. */ -static void slash_qdma_buf_sync_for_cpu(struct slash_qdma_buf *buf, - u64 start_entry, u64 n_entries, - enum dma_data_direction dir) +static void slash_qdma_buf_sync_sgl_for_cpu(struct slash_qdma_buf *buf, + const struct qdma_sw_sg *sgl, + u64 n_entries, + enum dma_data_direction dir) { struct device *dev = &buf->qdma_dev->pdev->dev; u64 i; for (i = 0; i < n_entries; i++) { - struct qdma_sw_sg *sg = &buf->sgl[start_entry + i]; + const struct qdma_sw_sg *sg = &sgl[i]; dma_sync_single_for_cpu(dev, sg->dma_addr, sg->len, dir); } @@ -2928,6 +2937,7 @@ static int slash_qdma_ioctl_buf_create_w(struct miscdevice *misc, * @qhndl: Resolved libqdma queue handle for the direction/qpair. * @start_entry: First page index of the buffer slice being transferred. * @n_entries: Number of pages in the slice (for the DMA sync). + * @xfer_sgl: Optional per-transfer SGL copy with a clipped final entry. * @dma_dir: DMA direction for the streaming sync calls. * @is_c2h: True for a C2H (device-to-host) sub-transfer, so the slice * is synced back for the CPU after completion. @@ -2946,6 +2956,7 @@ struct slash_qdma_xfer_req { unsigned long qhndl; u64 start_entry; u64 n_entries; + struct qdma_sw_sg *xfer_sgl; enum dma_data_direction dma_dir; bool is_c2h; unsigned int bytes_done; @@ -2988,8 +2999,9 @@ static int slash_qdma_xfer_done(struct qdma_request *qreq, * optional io_uring uring_cmd path. Resolves the buffer fd named by the * descriptor and refs the buffer, validates the slice against the buffer's * page granule and length, resolves the queue handle for the requested - * direction, syncs the pages touched by the slice for the device, and fills the cached, - * pre-DMA-mapped SGL slice into @xr->qreq (dma_mapped = 1, fp_done = NULL). + * direction, builds the per-transfer SGL slice, syncs the bytes touched by + * that slice for the device, and fills @xr->qreq (dma_mapped = 1, + * fp_done = NULL). * No pages are allocated or DMA-mapped here; that was amortised at creation. * * On success the caller owns the buffer ref in @xr->buf and must release it @@ -3003,6 +3015,7 @@ static int slash_qdma_xfer_prep(struct slash_qdma_dev *qdma_dev, struct slash_qdma_xfer_req *xr) { struct slash_qdma_buf *buf; + struct qdma_sw_sg *sgl; struct file *file; unsigned long qhndl; bool write; @@ -3088,20 +3101,41 @@ static int slash_qdma_xfer_prep(struct slash_qdma_dev *qdma_dev, mutex_unlock(&qdma_dev->lock); /* - * Hand the touched pages to the device. The mapping is persistent - * (dma_mapped = 1); only this slice is synced, so coherency cost scales - * with the transfer, not the whole buffer. + * Full-page transfers can point directly into the cached buffer SGL. + * Exact-length transfers with a partial final page need a per-transfer + * copy so libqdma sees the shortened tail descriptor without mutating the + * reusable buffer SGL. */ - slash_qdma_buf_sync_for_device(buf, start_entry, n_entries, dma_dir); + if ((desc->length % buf->granule) == 0) { + sgl = &buf->sgl[start_entry]; + } else { + u64 i; + u64 tail = desc->length % buf->granule; + + xr->xfer_sgl = kvmalloc_array(n_entries, sizeof(*xr->xfer_sgl), + GFP_KERNEL); + if (!xr->xfer_sgl) { + slash_qdma_buf_put(buf); + return -ENOMEM; + } + + for (i = 0; i < n_entries; i++) { + xr->xfer_sgl[i] = buf->sgl[start_entry + i]; + xr->xfer_sgl[i].next = + (i + 1 < n_entries) ? &xr->xfer_sgl[i + 1] : NULL; + } + xr->xfer_sgl[n_entries - 1].len = tail; + sgl = xr->xfer_sgl; + } /* - * Build the request from the cached SGL slice. dma_mapped = 1 tells - * libqdma the SGL is already DMA-mapped (dma_addr filled at creation), - * so it skips the per-request map/unmap entirely. + * Build the request from a cached or clipped SGL slice. dma_mapped = 1 + * tells libqdma the SGL is already DMA-mapped (dma_addr filled at buffer + * creation), so it skips the per-request map/unmap entirely. */ memset(&xr->qreq, 0, sizeof(xr->qreq)); xr->qreq.sgcnt = (unsigned int)n_entries; - xr->qreq.sgl = &buf->sgl[start_entry]; + xr->qreq.sgl = sgl; xr->qreq.write = write ? 1 : 0; xr->qreq.dma_mapped = 1; xr->qreq.udd_len = 0; @@ -3120,9 +3154,32 @@ static int slash_qdma_xfer_prep(struct slash_qdma_dev *qdma_dev, xr->bytes_done = 0; xr->err = 0; xr->async_inflight = false; + + /* + * Hand the touched bytes to the device. The mapping is persistent + * (dma_mapped = 1); only this transfer's SGL spans are synced, so + * coherency cost scales with the transfer, not the whole buffer. + */ + slash_qdma_buf_sync_sgl_for_device(buf, xr->qreq.sgl, n_entries, dma_dir); return 0; } +/** + * slash_qdma_xfer_cleanup() - Drop resources held by a prepared sub-transfer. + * @xr: Prepared transfer request. + * + * Releases the optional clipped SGL and the buffer reference taken in prep. + * It is safe for prepared-but-unsubmitted error paths. + */ +static void slash_qdma_xfer_cleanup(struct slash_qdma_xfer_req *xr) +{ + kvfree(xr->xfer_sgl); + xr->xfer_sgl = NULL; + if (xr->buf) + slash_qdma_buf_put(xr->buf); + xr->buf = NULL; +} + /** * slash_qdma_xfer_finish() - Post-completion DMA sync + buffer ref drop. * @xr: A prepared (and now completed) sub-transfer request. @@ -3133,9 +3190,9 @@ static int slash_qdma_xfer_prep(struct slash_qdma_dev *qdma_dev, static void slash_qdma_xfer_finish(struct slash_qdma_xfer_req *xr) { if (xr->is_c2h && xr->bytes_done) - slash_qdma_buf_sync_for_cpu(xr->buf, xr->start_entry, xr->n_entries, - xr->dma_dir); - slash_qdma_buf_put(xr->buf); + slash_qdma_buf_sync_sgl_for_cpu(xr->buf, xr->qreq.sgl, + xr->qreq.sgcnt, xr->dma_dir); + slash_qdma_xfer_cleanup(xr); } /** @@ -3203,7 +3260,7 @@ static long slash_qdma_qpair_transfer(struct file *file, void __user *uarg) &xrs[i]); if (rv) { while (i-- > 0) - slash_qdma_buf_put(xrs[i].buf); + slash_qdma_xfer_cleanup(&xrs[i]); kfree(xrs); return rv; } @@ -3426,7 +3483,7 @@ static int slash_qdma_qpair_uring_cmd(struct io_uring_cmd *cmd, &uc->xrs[i]); if (rv) { while (i-- > 0) - slash_qdma_buf_put(uc->xrs[i].buf); + slash_qdma_xfer_cleanup(&uc->xrs[i]); kfree(uc); return rv; } diff --git a/driver/tests/test_slash_qdma.c b/driver/tests/test_slash_qdma.c index 904b3e81..802f4d3b 100644 --- a/driver/tests/test_slash_qdma.c +++ b/driver/tests/test_slash_qdma.c @@ -373,6 +373,34 @@ TEST_F(qdma, qpair_add_cmpt_ring_size_out_of_range) EXPECT_EQ(EINVAL, errno); } +TEST_F(qdma, qpair_add_aperture_size_must_be_power_of_two) +{ + struct slash_qdma_qpair_add add; + + memset(&add, 0, sizeof(add)); + add.size = sizeof(add); + add.mode = 0; + add.dir_mask = 0x1; + add.aperture_size = 4097; + EXPECT_EQ(-1, ioctl(self->ctl_fd, SLASH_QDMA_IOCTL_QPAIR_ADD, &add)); + EXPECT_EQ(EINVAL, errno); +} + +TEST_F(qdma, qpair_add_keyhole_aperture) +{ + struct slash_qdma_qpair_add add; + + memset(&add, 0, sizeof(add)); + add.size = sizeof(add); + add.mode = 0; + add.dir_mask = 0x1; + add.aperture_size = 4096; + + ASSERT_EQ(0, ioctl(self->ctl_fd, SLASH_QDMA_IOCTL_QPAIR_ADD, &add)); + self->qid = add.qid; + self->qpair_added = 1; +} + TEST_F(qdma, q_op_invalid_op) { ASSERT_EQ(0, slash_qpair_add(self->ctl_fd, 0, 0x3, &self->qid)); @@ -816,24 +844,43 @@ TEST_F(qdma, qpair_get_fd_oversized_struct_zeros_tail) free(buf); } -TEST_F(qdma, reject_partial_4k_transfer) +TEST_F(qdma, partial_4k_write_read_verify) { - int buf_fd; + const size_t partial_size = TRANSFER_SIZE / 2; + int write_fd, read_fd; + uint8_t *write_buf, *read_buf; uint64_t dma_addr = get_dma_addr(); long ret; bring_up_qpair(_metadata, self, 0x3); - buf_fd = qdma_buf_create(self->io_fd, TRANSFER_SIZE, NULL, NULL); - ASSERT_GE(buf_fd, 0); + write_fd = qdma_buf_create(self->io_fd, TRANSFER_SIZE, NULL, NULL); + ASSERT_GE(write_fd, 0); + read_fd = qdma_buf_create(self->io_fd, TRANSFER_SIZE, NULL, NULL); + ASSERT_GE(read_fd, 0); - /* A sub-page length is not a multiple of the buffer granule. */ - ret = qdma_buf_transfer(self->io_fd, buf_fd, 0, dma_addr, - TRANSFER_SIZE / 2, SLASH_QDMA_XFER_H2C); - ASSERT_EQ(-1, ret); - ASSERT_EQ(EINVAL, errno); + write_buf = qdma_buf_map(write_fd, TRANSFER_SIZE); + ASSERT_NE(MAP_FAILED, write_buf); + read_buf = qdma_buf_map(read_fd, TRANSFER_SIZE); + ASSERT_NE(MAP_FAILED, read_buf); - close(buf_fd); + fill_pattern(write_buf, partial_size); + memset(read_buf, 0, TRANSFER_SIZE); + + ret = qdma_buf_transfer(self->io_fd, write_fd, 0, dma_addr, + partial_size, SLASH_QDMA_XFER_H2C); + ASSERT_EQ((ssize_t)partial_size, ret); + + ret = qdma_buf_transfer(self->io_fd, read_fd, 0, dma_addr, + partial_size, SLASH_QDMA_XFER_C2H); + ASSERT_EQ((ssize_t)partial_size, ret); + + EXPECT_EQ(0, memcmp(write_buf, read_buf, partial_size)); + + munmap(write_buf, TRANSFER_SIZE); + munmap(read_buf, TRANSFER_SIZE); + close(write_fd); + close(read_fd); } TEST_F(qdma, multipage_4k_write_read_verify) diff --git a/vrt/vrtd/src/design_writer.c b/vrt/vrtd/src/design_writer.c index 25de7695..cacf591f 100644 --- a/vrt/vrtd/src/design_writer.c +++ b/vrt/vrtd/src/design_writer.c @@ -65,10 +65,12 @@ * ---------------------- * At creation time, design_writer_open_qpair() allocates a QDMA queue pair in * Host-to-Card (H2C) memory-mapped mode, starts it, and obtains a file - * descriptor for the queue. The bitstream is then copied into a kernel-owned - * QDMA buffer and submitted with slash_qdma_qpair_transfer() at offset - * VRTD_DESIGN_WRITER_SEEK_ADDR (0x102100000), which the QDMA subsystem - * translates into DMA writes to the FPGA's configuration memory. + * descriptor for the queue. The queue is configured in libqdma keyhole mode + * so endpoint addresses wrap inside the PDI ingress aperture. The bitstream + * is then copied into a kernel-owned QDMA buffer and submitted with + * slash_qdma_qpair_transfer() at offset VRTD_DESIGN_WRITER_SEEK_ADDR + * (0x102100000), which the QDMA subsystem translates into DMA writes to the + * FPGA's configuration memory. * * Error propagation * ----------------- @@ -118,12 +120,8 @@ */ #define VRTD_DESIGN_WRITER_SEEK_ADDR 0x102100000ull -/* - * The PDI boot stream is exposed as a 64 KiB aperture in the CIPS address map. - * Large PDI payloads must be fed through that aperture instead of linearly - * walking past it into unmapped NoC address space. - */ -#define VRTD_DESIGN_WRITER_APERTURE_BYTES 0x10000ull +/* Previously-working libqdma keyhole aperture for PDI delivery. */ +#define VRTD_DESIGN_WRITER_APERTURE_BYTES 0x1000u /* Maximum bitstream size accepted by the design writer (1 GiB). */ #define VRTD_DESIGN_WRITER_MAX_BYTES (1ull * 1024 * 1024 * 1024) // 1 GiB @@ -257,12 +255,12 @@ static ssize_t read_entire_file(int fd, void **bufp) } /** - * Transfer the full contents of a buffer through the programming aperture. + * Transfer the full contents of a buffer through the programming keyhole. * * Copies the userspace bitstream into a kernel-owned QDMA buffer, then submits - * H2C MM transfers through the design writer queue-pair fd. The device-side - * address wraps inside the boot-stream aperture so multi-megabyte PDIs do not - * walk past the 64 KiB mapped programming window. + * one H2C MM transfer through the design writer queue-pair fd. The queue is + * configured with a keyhole aperture, so libqdma wraps endpoint addresses as + * it consumes the multi-megabyte PDI stream. * * @param writer The design writer instance. * @param buf Source buffer containing bitstream data. @@ -287,35 +285,23 @@ static int transfer_all_at_pos(struct design_writer *writer, const void *buf, si LOG( LOG_INFO, - "Transferring design writer payload to device offset 0x%llx (bytes=%zu buffer_bytes=%llu aperture_bytes=%llu)", + "Transferring design writer payload to device offset 0x%llx (bytes=%zu buffer_bytes=%llu keyhole_bytes=%u)", (unsigned long long)pos, len, (unsigned long long)dma_len, - (unsigned long long)VRTD_DESIGN_WRITER_APERTURE_BYTES + VRTD_DESIGN_WRITER_APERTURE_BYTES ); - uint64_t off = 0; - while (off < (uint64_t)len) { - uint64_t aperture_offset = off % VRTD_DESIGN_WRITER_APERTURE_BYTES; - uint64_t chunk = VRTD_DESIGN_WRITER_APERTURE_BYTES - aperture_offset; - uint64_t remaining = (uint64_t)len - off; - if (chunk > remaining) { - chunk = remaining; - } - - ssize_t transferred = slash_qdma_qpair_transfer( - writer->fd, - dma_buf.fd, - off, - pos + aperture_offset, - chunk, - SLASH_QDMA_XFER_H2C - ); - PROPAGATE_ERROR_STDC_LOG(transferred, LOG_ERR, "Failed to transfer design writer payload"); - if ((uint64_t)transferred != chunk) { - errno = EIO; - PROPAGATE_ERROR_STDC_LOG(-1, LOG_ERR, "Short design writer transfer"); - } - - off += chunk; + ssize_t transferred = slash_qdma_qpair_transfer( + writer->fd, + dma_buf.fd, + 0, + pos, + (uint64_t)len, + SLASH_QDMA_XFER_H2C + ); + PROPAGATE_ERROR_STDC_LOG(transferred, LOG_ERR, "Failed to transfer design writer payload"); + if ((uint64_t)transferred != (uint64_t)len) { + errno = EIO; + PROPAGATE_ERROR_STDC_LOG(-1, LOG_ERR, "Short design writer transfer"); } return 0; @@ -613,6 +599,7 @@ static int design_writer_open_qpair(struct design_writer *writer) qpair.h2c_ring_sz = VRTD_QDMA_RING_SZ_IDX; qpair.c2h_ring_sz = VRTD_QDMA_RING_SZ_IDX; qpair.cmpt_ring_sz = VRTD_QDMA_RING_SZ_IDX; + qpair.aperture_size = VRTD_DESIGN_WRITER_APERTURE_BYTES; int ret = slash_qdma_qpair_add(writer->qdma, &qpair); PROPAGATE_ERROR_STDC_LOG(ret, LOG_ERR, "Failed to add design writer QDMA qpair");