diff --git a/arch/arm/include/asm/arch-apple/rtkit.h b/arch/arm/include/asm/arch-apple/rtkit.h index 51f77f298c..eff18ddb9d 100644 --- a/arch/arm/include/asm/arch-apple/rtkit.h +++ b/arch/arm/include/asm/arch-apple/rtkit.h @@ -7,5 +7,23 @@ #define APPLE_RTKIT_PWR_STATE_QUIESCED 0x10 #define APPLE_RTKIT_PWR_STATE_ON 0x20 -int apple_rtkit_init(struct mbox_chan *); -int apple_rtkit_shutdown(struct mbox_chan *, int); +struct apple_rtkit_buffer { + void *buffer; + u64 dva; + size_t size; + bool is_mapped; +}; + +typedef int (*apple_rtkit_shmem_setup)(void *cookie, + struct apple_rtkit_buffer *buf); +typedef void (*apple_rtkit_shmem_destroy)(void *cookie, + struct apple_rtkit_buffer *buf); + +struct apple_rtkit; + +struct apple_rtkit *apple_rtkit_init(struct mbox_chan *chan, void *cookie, + apple_rtkit_shmem_setup shmem_setup, + apple_rtkit_shmem_destroy shmem_destroy); +void apple_rtkit_free(struct apple_rtkit *rtk); +int apple_rtkit_boot(struct apple_rtkit *rtk); +int apple_rtkit_shutdown(struct apple_rtkit *rtk, int pwrstate); diff --git a/arch/arm/include/asm/arch-apple/sart.h b/arch/arm/include/asm/arch-apple/sart.h new file mode 100644 index 0000000000..b99bdeafc0 --- /dev/null +++ b/arch/arm/include/asm/arch-apple/sart.h @@ -0,0 +1,22 @@ +/* SPDX-License-Identifier: MIT + * + * The sart code is copied from m1n1 (https://github.com/AsahiLinux/m1n1) and + * licensed as MIT. + * + * (C) Copyright 2022 The Asahi Linux Contributors + */ + +#ifndef SART_H +#define SART_H + +#include + +struct apple_sart; + +struct apple_sart *sart_init(ofnode node); +void sart_free(struct apple_sart *sart); + +bool sart_add_allowed_region(struct apple_sart *sart, void *paddr, size_t sz); +bool sart_remove_allowed_region(struct apple_sart *sart, void *paddr, size_t sz); + +#endif diff --git a/arch/arm/mach-apple/Makefile b/arch/arm/mach-apple/Makefile index 52f30a777b..50b465b947 100644 --- a/arch/arm/mach-apple/Makefile +++ b/arch/arm/mach-apple/Makefile @@ -3,3 +3,4 @@ obj-y += board.o obj-y += lowlevel_init.o obj-y += rtkit.o +obj-$(CONFIG_NVME_APPLE) += sart.o diff --git a/arch/arm/mach-apple/rtkit.c b/arch/arm/mach-apple/rtkit.c index 2dcb8bdd3e..da77718442 100644 --- a/arch/arm/mach-apple/rtkit.c +++ b/arch/arm/mach-apple/rtkit.c @@ -17,6 +17,7 @@ #define APPLE_RTKIT_EP_SYSLOG 2 #define APPLE_RTKIT_EP_DEBUG 3 #define APPLE_RTKIT_EP_IOREPORT 4 +#define APPLE_RTKIT_EP_TRACEKIT 10 /* Messages for management endpoint. */ #define APPLE_RTKIT_MGMT_TYPE GENMASK(59, 52) @@ -51,7 +52,102 @@ #define APPLE_RTKIT_BUFFER_REQUEST_SIZE GENMASK(51, 44) #define APPLE_RTKIT_BUFFER_REQUEST_IOVA GENMASK(41, 0) -int apple_rtkit_init(struct mbox_chan *chan) +struct apple_rtkit { + struct mbox_chan *chan; + void *cookie; + apple_rtkit_shmem_setup shmem_setup; + apple_rtkit_shmem_destroy shmem_destroy; + + struct apple_rtkit_buffer syslog_buffer; + struct apple_rtkit_buffer crashlog_buffer; + struct apple_rtkit_buffer ioreport_buffer; +}; + +struct apple_rtkit *apple_rtkit_init(struct mbox_chan *chan, void *cookie, + apple_rtkit_shmem_setup shmem_setup, + apple_rtkit_shmem_destroy shmem_destroy) +{ + struct apple_rtkit *rtk; + + rtk = calloc(sizeof(*rtk), 1); + if (!rtk) + return NULL; + + rtk->chan = chan; + rtk->cookie = cookie; + rtk->shmem_setup = shmem_setup; + rtk->shmem_destroy = shmem_destroy; + + return rtk; +} + +void apple_rtkit_free(struct apple_rtkit *rtk) +{ + if (rtk->shmem_destroy) { + if (rtk->syslog_buffer.buffer) + rtk->shmem_destroy(rtk->cookie, &rtk->syslog_buffer); + if (rtk->crashlog_buffer.buffer) + rtk->shmem_destroy(rtk->cookie, &rtk->crashlog_buffer); + if (rtk->ioreport_buffer.buffer) + rtk->shmem_destroy(rtk->cookie, &rtk->ioreport_buffer); + } + free(rtk); +} + +static int rtkit_handle_buf_req(struct apple_rtkit *rtk, int endpoint, struct apple_mbox_msg *msg) +{ + struct apple_rtkit_buffer *buf; + size_t num_4kpages; + int ret; + + num_4kpages = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg->msg0); + + if (num_4kpages == 0) { + printf("%s: unexpected request for buffer without size\n", __func__); + return -1; + } + + switch (endpoint) { + case APPLE_RTKIT_EP_CRASHLOG: + buf = &rtk->crashlog_buffer; + break; + case APPLE_RTKIT_EP_SYSLOG: + buf = &rtk->syslog_buffer; + break; + case APPLE_RTKIT_EP_IOREPORT: + buf = &rtk->ioreport_buffer; + break; + default: + printf("%s: unexpected endpoint %d\n", __func__, endpoint); + return -1; + } + + buf->dva = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg->msg0); + buf->size = num_4kpages << 12; + buf->is_mapped = false; + + if (rtk->shmem_setup) { + ret = rtk->shmem_setup(rtk->cookie, buf); + if (ret < 0) { + printf("%s: shmen_setup failed for endpoint %d\n", __func__, + endpoint); + return ret; + } + } + + if (!buf->is_mapped) { + msg->msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_BUFFER_REQUEST) | + FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, num_4kpages) | + FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, buf->dva); + msg->msg1 = endpoint; + + return mbox_send(rtk->chan, msg); + } + + return 0; +} + +int apple_rtkit_boot(struct apple_rtkit *rtk) { struct apple_mbox_msg msg; int endpoints[256]; @@ -67,12 +163,12 @@ int apple_rtkit_init(struct mbox_chan *chan) msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE) | FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, APPLE_RTKIT_PWR_STATE_ON); msg.msg1 = APPLE_RTKIT_EP_MGMT; - ret = mbox_send(chan, &msg); + ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret; /* Wait for protocol version negotiation message. */ - ret = mbox_recv(chan, &msg, 10000); + ret = mbox_recv(rtk->chan, &msg, 10000); if (ret < 0) return ret; @@ -108,13 +204,13 @@ int apple_rtkit_init(struct mbox_chan *chan) FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MINVER, want_ver) | FIELD_PREP(APPLE_RTKIT_MGMT_HELLO_MAXVER, want_ver); msg.msg1 = APPLE_RTKIT_EP_MGMT; - ret = mbox_send(chan, &msg); + ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret; wait_epmap: /* Wait for endpoint map message. */ - ret = mbox_recv(chan, &msg, 10000); + ret = mbox_recv(rtk->chan, &msg, 10000); if (ret < 0) return ret; @@ -145,7 +241,7 @@ wait_epmap: reply |= APPLE_RTKIT_MGMT_EPMAP_REPLY_MORE; msg.msg0 = reply; msg.msg1 = APPLE_RTKIT_EP_MGMT; - ret = mbox_send(chan, &msg); + ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret; @@ -153,24 +249,33 @@ wait_epmap: goto wait_epmap; for (i = 0; i < nendpoints; i++) { - /* Don't start the syslog endpoint since we can't - easily handle its messages in U-Boot. */ - if (endpoints[i] == APPLE_RTKIT_EP_SYSLOG) + /* Start only necessary endpoints. The syslog endpoint is + * particularly noisy and its message can't easily be handled + * within U-Boot. + */ + switch (endpoints[i]) { + case APPLE_RTKIT_EP_MGMT: + case APPLE_RTKIT_EP_SYSLOG: + case APPLE_RTKIT_EP_DEBUG: + case APPLE_RTKIT_EP_TRACEKIT: continue; + default: + break; + } /* Request endpoint. */ msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_STARTEP) | FIELD_PREP(APPLE_RTKIT_MGMT_STARTEP_EP, endpoints[i]) | APPLE_RTKIT_MGMT_STARTEP_FLAG; msg.msg1 = APPLE_RTKIT_EP_MGMT; - ret = mbox_send(chan, &msg); + ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret; } pwrstate = APPLE_RTKIT_PWR_STATE_SLEEP; while (pwrstate != APPLE_RTKIT_PWR_STATE_ON) { - ret = mbox_recv(chan, &msg, 1000000); + ret = mbox_recv(rtk->chan, &msg, 100000); if (ret < 0) return ret; @@ -180,20 +285,22 @@ wait_epmap: if (endpoint == APPLE_RTKIT_EP_CRASHLOG || endpoint == APPLE_RTKIT_EP_SYSLOG || endpoint == APPLE_RTKIT_EP_IOREPORT) { - u64 addr = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_IOVA, msg.msg0); - u64 size = FIELD_GET(APPLE_RTKIT_BUFFER_REQUEST_SIZE, msg.msg0); - - if (msgtype == APPLE_RTKIT_BUFFER_REQUEST && addr != 0) + if (msgtype == APPLE_RTKIT_BUFFER_REQUEST) { + ret = rtkit_handle_buf_req(rtk, endpoint, &msg); + if (ret < 0) + return ret; continue; + } + } - msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_BUFFER_REQUEST) | - FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_SIZE, size) | - FIELD_PREP(APPLE_RTKIT_BUFFER_REQUEST_IOVA, addr); - msg.msg1 = endpoint; - ret = mbox_send(chan, &msg); - if (ret < 0) - return ret; - continue; + if (endpoint == APPLE_RTKIT_EP_IOREPORT) { + // these two messages have to be ack-ed for proper startup + if (msgtype == 0xc || msgtype == 0x8) { + ret = mbox_send(rtk->chan, &msg); + if (ret < 0) + return ret; + continue; + } } if (endpoint != APPLE_RTKIT_EP_MGMT) { @@ -211,7 +318,7 @@ wait_epmap: return 0; } -int apple_rtkit_shutdown(struct mbox_chan *chan, int pwrstate) +int apple_rtkit_shutdown(struct apple_rtkit *rtk, int pwrstate) { struct apple_mbox_msg msg; int ret; @@ -219,11 +326,11 @@ int apple_rtkit_shutdown(struct mbox_chan *chan, int pwrstate) msg.msg0 = FIELD_PREP(APPLE_RTKIT_MGMT_TYPE, APPLE_RTKIT_MGMT_SET_IOP_PWR_STATE) | FIELD_PREP(APPLE_RTKIT_MGMT_PWR_STATE, pwrstate); msg.msg1 = APPLE_RTKIT_EP_MGMT; - ret = mbox_send(chan, &msg); + ret = mbox_send(rtk->chan, &msg); if (ret < 0) return ret; - ret = mbox_recv(chan, &msg, 100000); + ret = mbox_recv(rtk->chan, &msg, 100000); if (ret < 0) return ret; diff --git a/arch/arm/mach-apple/sart.c b/arch/arm/mach-apple/sart.c new file mode 100644 index 0000000000..e9b017ad57 --- /dev/null +++ b/arch/arm/mach-apple/sart.c @@ -0,0 +1,230 @@ +// SPDX-License-Identifier: MIT +/* + * The sart code is copied from m1n1 (https://github.com/AsahiLinux/m1n1) and + * licensed as MIT. + * + * (C) Copyright 2022 The Asahi Linux Contributors + */ + +#include +#include + +#include +#include + +#include + +#define APPLE_SART_MAX_ENTRIES 16 + +/* This is probably a bitfield but the exact meaning of each bit is unknown. */ +#define APPLE_SART_FLAGS_ALLOW 0xff + +/* SARTv2 registers */ +#define APPLE_SART2_CONFIG(idx) (0x00 + 4 * (idx)) +#define APPLE_SART2_CONFIG_FLAGS GENMASK(31, 24) +#define APPLE_SART2_CONFIG_SIZE GENMASK(23, 0) +#define APPLE_SART2_CONFIG_SIZE_SHIFT 12 +#define APPLE_SART2_CONFIG_SIZE_MAX GENMASK(23, 0) + +#define APPLE_SART2_PADDR(idx) (0x40 + 4 * (idx)) +#define APPLE_SART2_PADDR_SHIFT 12 + +/* SARTv3 registers */ +#define APPLE_SART3_CONFIG(idx) (0x00 + 4 * (idx)) + +#define APPLE_SART3_PADDR(idx) (0x40 + 4 * (idx)) +#define APPLE_SART3_PADDR_SHIFT 12 + +#define APPLE_SART3_SIZE(idx) (0x80 + 4 * (idx)) +#define APPLE_SART3_SIZE_SHIFT 12 +#define APPLE_SART3_SIZE_MAX GENMASK(29, 0) + +struct apple_sart { + uintptr_t base; + u32 protected_entries; + + void (*get_entry)(struct apple_sart *sart, int index, u8 *flags, void **paddr, + size_t *size); + bool (*set_entry)(struct apple_sart *sart, int index, u8 flags, void *paddr, + size_t size); +}; + +static void sart2_get_entry(struct apple_sart *sart, int index, u8 *flags, void **paddr, + size_t *size) +{ + u32 cfg = readl(sart->base + APPLE_SART2_CONFIG(index)); + *flags = FIELD_GET(APPLE_SART2_CONFIG_FLAGS, cfg); + *size = (size_t)FIELD_GET(APPLE_SART2_CONFIG_SIZE, cfg) << APPLE_SART2_CONFIG_SIZE_SHIFT; + *paddr = (void *) + ((u64)readl(sart->base + APPLE_SART2_PADDR(index)) << APPLE_SART2_PADDR_SHIFT); +} + +static bool sart2_set_entry(struct apple_sart *sart, int index, u8 flags, void *paddr_, + size_t size) +{ + u32 cfg; + u64 paddr = (u64)paddr_; + + if (size & ((1 << APPLE_SART2_CONFIG_SIZE_SHIFT) - 1)) + return false; + if (paddr & ((1 << APPLE_SART2_PADDR_SHIFT) - 1)) + return false; + + size >>= APPLE_SART2_CONFIG_SIZE_SHIFT; + paddr >>= APPLE_SART2_PADDR_SHIFT; + + if (size > APPLE_SART2_CONFIG_SIZE_MAX) + return false; + + cfg = FIELD_PREP(APPLE_SART2_CONFIG_FLAGS, flags); + cfg |= FIELD_PREP(APPLE_SART2_CONFIG_SIZE, size); + + writel(paddr, sart->base + APPLE_SART2_PADDR(index)); + writel(cfg, sart->base + APPLE_SART2_CONFIG(index)); + + return true; +} + +static void sart3_get_entry(struct apple_sart *sart, int index, u8 *flags, void **paddr, + size_t *size) +{ + *flags = readl(sart->base + APPLE_SART3_CONFIG(index)); + *size = (size_t)readl(sart->base + APPLE_SART3_SIZE(index)) << APPLE_SART3_SIZE_SHIFT; + *paddr = (void *) + ((u64)readl(sart->base + APPLE_SART3_PADDR(index)) << APPLE_SART3_PADDR_SHIFT); +} + +static bool sart3_set_entry(struct apple_sart *sart, int index, u8 flags, void *paddr_, + size_t size) +{ + u64 paddr = (u64)paddr_; + + if (size & ((1 << APPLE_SART3_SIZE_SHIFT) - 1)) + return false; + if (paddr & ((1 << APPLE_SART3_PADDR_SHIFT) - 1)) + return false; + + paddr >>= APPLE_SART3_PADDR_SHIFT; + size >>= APPLE_SART3_SIZE_SHIFT; + + if (size > APPLE_SART3_SIZE_MAX) + return false; + + writel(paddr, sart->base + APPLE_SART3_PADDR(index)); + writel(size, sart->base + APPLE_SART3_SIZE(index)); + writel(flags, sart->base + APPLE_SART3_CONFIG(index)); + + return true; +} + +struct apple_sart *sart_init(ofnode node) +{ + phys_addr_t base; + u32 sart_version; + struct apple_sart *sart; + + base = ofnode_get_addr(node); + if (base == FDT_ADDR_T_NONE) + return NULL; + + if (ofnode_device_is_compatible(node, "apple,t8103-sart")) { + sart_version = 2; + } else if (ofnode_device_is_compatible(node, "apple,t6000-sart")) { + sart_version = 3; + } else { + printf("sart: unknown SART compatible: %sd\n", + ofnode_read_string(node, "compatible")); + return NULL; + } + + sart = calloc(sizeof(*sart), 1); + if (!sart) + return NULL; + + sart->base = base; + + switch (sart_version) { + case 2: + sart->get_entry = sart2_get_entry; + sart->set_entry = sart2_set_entry; + break; + case 3: + sart->get_entry = sart3_get_entry; + sart->set_entry = sart3_set_entry; + break; + default: + printf("sart: SART has unknown version %d\n", sart_version); + free(sart); + return NULL; + } + + sart->protected_entries = 0; + for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { + void *paddr; + u8 flags; + size_t sz; + + sart->get_entry(sart, i, &flags, &paddr, &sz); + if (flags) + sart->protected_entries |= 1 << i; + } + + return sart; +} + +void sart_free(struct apple_sart *sart) +{ + for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { + if (sart->protected_entries & (1 << i)) + continue; + sart->set_entry(sart, i, 0, NULL, 0); + } + + free(sart); +} + +bool sart_add_allowed_region(struct apple_sart *sart, void *paddr, size_t sz) +{ + for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { + void *e_paddr; + u8 e_flags; + size_t e_sz; + + if (sart->protected_entries & (1 << i)) + continue; + + sart->get_entry(sart, i, &e_flags, &e_paddr, &e_sz); + if (e_flags) + continue; + + return sart->set_entry(sart, i, APPLE_SART_FLAGS_ALLOW, paddr, sz); + } + + printf("sart: no more free entries\n"); + return false; +} + +bool sart_remove_allowed_region(struct apple_sart *sart, void *paddr, size_t sz) +{ + for (unsigned int i = 0; i < APPLE_SART_MAX_ENTRIES; ++i) { + void *e_paddr; + u8 e_flags; + size_t e_sz; + + if (sart->protected_entries & (1 << i)) + continue; + + sart->get_entry(sart, i, &e_flags, &e_paddr, &e_sz); + if (!e_flags) + continue; + if (e_paddr != paddr) + continue; + if (e_sz != sz) + continue; + + return sart->set_entry(sart, i, 0, NULL, 0); + } + + printf("sart: could not find entry to be removed\n"); + return false; +} diff --git a/drivers/nvme/nvme_apple.c b/drivers/nvme/nvme_apple.c index d9d491c2be..819b748dc0 100644 --- a/drivers/nvme/nvme_apple.c +++ b/drivers/nvme/nvme_apple.c @@ -12,6 +12,7 @@ #include #include +#include #include /* ASC registers */ @@ -66,6 +67,8 @@ struct apple_nvme_priv { void *asc; /* ASC registers */ struct reset_ctl_bulk resets; /* ASC reset */ struct mbox_chan chan; + struct apple_sart *sart; + struct apple_rtkit *rtk; struct ans_nvmmu_tcb *tcbs[NVME_Q_NUM]; /* Submission queue TCBs */ u32 __iomem *q_db[NVME_Q_NUM]; /* Submission queue doorbell */ }; @@ -143,11 +146,51 @@ static void apple_nvme_complete_cmd(struct nvme_queue *nvmeq, nvmeq->sq_tail = tail; } +static int nvme_shmem_setup(void *cookie, struct apple_rtkit_buffer *buf) +{ + struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie; + + if (!buf || buf->dva || !buf->size) + return -1; + + buf->buffer = memalign(SZ_16K, ALIGN(buf->size, SZ_16K)); + if (!buf->buffer) + return -ENOMEM; + + if (!sart_add_allowed_region(priv->sart, buf->buffer, buf->size)) { + free(buf->buffer); + buf->buffer = NULL; + buf->size = 0; + return -1; + } + + buf->dva = (u64)buf->buffer; + + return 0; +} + +static void nvme_shmem_destroy(void *cookie, struct apple_rtkit_buffer *buf) +{ + struct apple_nvme_priv *priv = (struct apple_nvme_priv *)cookie; + + if (!buf) + return; + + if (buf->buffer) { + sart_remove_allowed_region(priv->sart, buf->buffer, buf->size); + free(buf->buffer); + buf->buffer = NULL; + buf->size = 0; + buf->dva = 0; + } +} + static int apple_nvme_probe(struct udevice *dev) { struct apple_nvme_priv *priv = dev_get_priv(dev); fdt_addr_t addr; - u32 ctrl, stat; + ofnode of_sart; + u32 ctrl, stat, phandle; int ret; priv->base = dev_read_addr_ptr(dev); @@ -167,12 +210,27 @@ static int apple_nvme_probe(struct udevice *dev) if (ret < 0) return ret; + ret = dev_read_u32(dev, "apple,sart", &phandle); + if (ret < 0) + return ret; + + of_sart = ofnode_get_by_phandle(phandle); + priv->sart = sart_init(of_sart); + if (!priv->sart) + return -EINVAL; + ctrl = readl(priv->asc + REG_CPU_CTRL); writel(ctrl | REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL); - ret = apple_rtkit_init(&priv->chan); - if (ret < 0) + priv->rtk = apple_rtkit_init(&priv->chan, priv, nvme_shmem_setup, nvme_shmem_destroy); + if (!priv->rtk) + return -ENOMEM; + + ret = apple_rtkit_boot(priv->rtk); + if (ret < 0) { + printf("%s: NVMe apple_rtkit_boot returned: %d\n", __func__, ret); return ret; + } ret = readl_poll_sleep_timeout(priv->base + ANS_BOOT_STATUS, stat, (stat == ANS_BOOT_STATUS_OK), 100, @@ -206,11 +264,17 @@ static int apple_nvme_remove(struct udevice *dev) nvme_shutdown(dev); - apple_rtkit_shutdown(&priv->chan, APPLE_RTKIT_PWR_STATE_SLEEP); + apple_rtkit_shutdown(priv->rtk, APPLE_RTKIT_PWR_STATE_SLEEP); ctrl = readl(priv->asc + REG_CPU_CTRL); writel(ctrl & ~REG_CPU_CTRL_RUN, priv->asc + REG_CPU_CTRL); + apple_rtkit_free(priv->rtk); + priv->rtk = NULL; + + sart_free(priv->sart); + priv->sart = NULL; + reset_assert_bulk(&priv->resets); reset_deassert_bulk(&priv->resets);