diff --git a/drivers/nvme/host/core.c b/drivers/nvme/host/core.c index cc23035148b4..6f3b787e2336 100644 --- a/drivers/nvme/host/core.c +++ b/drivers/nvme/host/core.c @@ -4065,7 +4065,41 @@ static void nvme_validate_ns(struct nvme_ns *ns, struct nvme_ns_info *info) nvme_ns_remove(ns); } -static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid) +static void nvme_scan_validate_ns_work(struct work_struct *work) +{ + struct nvme_ns *ns; + struct async_scan_data *data = container_of(work, struct nvme_ns_info, work); + + sync_synchronize_full_domain(&data->domain); + + ns = nvme_find_get_ns(data->ctrl, data->info.nsid); + if (ns) { + nvme_validate_ns(ns, &data->info); + nvme_put_ns(ns); + } else { + nvme_alloc_ns(data->ctrl, &data->info); + } + kfree(data); +} + +static void nvme_scan_ns(void *data, async_cookie_t cookie) +{ + ASYNC_DOMAIN(domain); + + struct async_scan_data *data = kzalloc(sizeof(*data)); + if (!info) + return; + + data->info.nsid = nsid; + data->async_domain = domain; + INIT_WORK(info->work, nvme_scan_validate_ns_work); + + async_schedule_domain(nvme_scan_ns_async, data, data->domain); + queue_work(data->wq, &data->work); +} + +static void nvme_scan_ns_async(struct nvme_ctrl *ctrl, unsigned nsid + struct async_scan_data *async_data) { struct nvme_ns_info info = { .nsid = nsid }; struct nvme_ns *ns; @@ -4111,33 +4145,42 @@ static void nvme_scan_ns(struct nvme_ctrl *ctrl, unsigned nsid) } } -/** - * struct async_scan_info - keeps track of controller & NSIDs to scan - * @ctrl: Controller on which namespaces are being scanned - * @next_nsid: Index of next NSID to scan in ns_list - * @ns_list: Pointer to list of NSIDs to scan - * - * Note: There is a single async_scan_info structure shared by all instances - * of nvme_scan_ns_async() scanning a given controller, so the atomic - * operations on next_nsid are critical to ensure each instance scans a unique - * NSID. - */ -struct async_scan_info { +struct async_scan_data { + struct nvme_ns_info info; struct nvme_ctrl *ctrl; - atomic_t next_nsid; - __le32 *ns_list; + struct async_domain *async_domain; + struct workqueue_struct *wq; }; -static void nvme_scan_ns_async(void *data, async_cookie_t cookie) +static struct async_scan_data *nvme_async_scan_init(struct nvme_ctrl *ctrl, + struct async_domain *d) { - struct async_scan_info *scan_info = data; - int idx; - u32 nsid; + struct async_scan_data *data; - idx = (u32)atomic_fetch_inc(&scan_info->next_nsid); - nsid = le32_to_cpu(scan_info->ns_list[idx]); + data = kmalloc(sizeof(*data)); + if (!data) + return NULL; - nvme_scan_ns(scan_info->ctrl, nsid); + data->ctrl = ctrl; + data->async_domain = d; + data->wq = alloc_ordered_workqueue("nvme_async_wq", 0); + if (!data->wq) { + kfree(data); + return NULL; + } + + return data; +} + +static void nvme_async_scan_sync(struct async_scan_data *data) +{ + flush_workqueue(data->wq); +} + +static void nvme_async_scan_finish(struct async_scan_data *data) +{ + destroy_workqueue(data->wq); + kfree(data); } static void nvme_remove_invalid_namespaces(struct nvme_ctrl *ctrl, @@ -4167,14 +4210,17 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) u32 prev = 0; int ret = 0, i; ASYNC_DOMAIN(domain); - struct async_scan_info scan_info; + struct async_scan_data *scan_data; ns_list = kzalloc(NVME_IDENTIFY_DATA_SIZE, GFP_KERNEL); if (!ns_list) return -ENOMEM; - scan_info.ctrl = ctrl; - scan_info.ns_list = ns_list; + scan_data = nvme_async_scan_init(ctrl, &domain); + if (!scan_data) { + kfree(ns_list); + return -ENOMEM; + } for (;;) { struct nvme_command cmd = { .identify.opcode = nvme_admin_identify, @@ -4201,12 +4247,12 @@ static int nvme_scan_ns_list(struct nvme_ctrl *ctrl) while (++prev < nsid) nvme_ns_remove_by_nsid(ctrl, prev); } - async_synchronize_full_domain(&domain); } + nvme_async_scan_sync(scan_data); out: nvme_remove_invalid_namespaces(ctrl, prev); free: - async_synchronize_full_domain(&domain); + nvme_async_scan_finish(scan_data); kfree(ns_list); return ret; } @@ -4215,15 +4261,23 @@ static void nvme_scan_ns_sequential(struct nvme_ctrl *ctrl) { struct nvme_id_ctrl *id; u32 nn, i; + struct async_scan_data *scan_data; + ASYNC_DOMAIN(domain); if (nvme_identify_ctrl(ctrl, &id)) return; + nn = le32_to_cpu(id->nn); kfree(id); + scan_data = nvme_async_scan_init(ctrl, &domain) + if (!data) + return; + for (i = 1; i <= nn; i++) - nvme_scan_ns(ctrl, i); + nvme_scan_ns(ctrl, i, scan_data); + nvme_async_scan_finish(scan_data); nvme_remove_invalid_namespaces(ctrl, nn); }