// SPDX-License-Identifier: GPL-2.0
/*
* Bring a ublk device live while some of its fetched io commands are
* canceled.
*
* A cancel completes a fetched command and clears io->cmd, but the io
* still counts as ready. Only ubq->canceling keeps ublk_queue_rq() away
* from the NULL io->cmd. Each mode below loses ->canceling in another way:
*
* stop_start: fetch every tag, STOP_DEV before START_DEV, START_DEV.
* STOP_DEV cancels the commands of the attached server,
* so START_DEV must get -EBUSY until that server is gone.
* An unfixed kernel starts the device over them.
*
* partial_fetch: task A fetches tags 0..depth-2 and dies, so its
* commands are canceled. Task B fetches the last tag, and
* an unfixed kernel clears ->canceling when the queue gets
* ready. /dev/ublkcN stays open all the time, so
* ublk_ch_release() never resets the queue.
*
* recovery: a UBLK_F_USER_RECOVERY device with two queues loses its
* server. During recovery task Q0 fetches queue 0, which
* clears q0->canceling, then dies. An unfixed kernel still
* has ub->canceling set because queue 1 is not ready, so
* ublk_start_cancel() does not mark queue 0 again. Reads
* are issued on a CPU mapped to queue 0.
*
* In partial_fetch and recovery, START_DEV / END_USER_RECOVERY must refuse
* with -EBUSY. An unfixed kernel oopses in ublk_queue_cmd() on a NULL
* io->cmd.
*
* stop_restart: control mode. STOP_DEV on a new device, before any
* server opened it, takes no command. A server started
* afterwards has to start the device and serve I/O.
*
* stop_attached: STOP_DEV on a device whose server opened it but fetched
* nothing stops that server too: FETCH gets ABORT and
* START_DEV -EBUSY, until a new server opens the device.
*
* stop_live_restart: STOP_DEV on a live device; once its server is gone,
* a new server has to fetch and start it again. An unfixed
* ublk_ch_release() skips the reset once the disk is gone,
* so the new FETCH gets -EBUSY.
*
* race_start: STOP_DEV and START_DEV at the same time on a device whose
* server fetched every tag. START_DEV either wins, and
* reads complete (they fail once STOP_DEV removes the
* disk), or gets -EBUSY. An unfixed kernel cancels the
* commands of the live disk.
*
* race_fetch: STOP_DEV while a server opens the device and fetches,
* then START_DEV. If STOP_DEV came before the open, it must
* not take any command, so START_DEV works and every read
* succeeds; otherwise START_DEV gets -EBUSY. An unfixed
* kernel takes commands fetched after its unlock and goes
* live over them.
*
* race_async_fetch: FETCH with IOSQE_ASYNC while STOP_DEV cancels,
* then close the ring. An unfixed FETCH marks its command
* cancelable only after publishing it and dropping
* ub->mutex; a cancel in between completes a command which
* is then put on io_uring's cancelable list. Closing the
* ring walks that list: KASAN reports a use after free.
* Needs a KASAN kernel to see the bug; otherwise it must
* just not crash.
*
* A dying task is a child process which fetches and then calls exec().
* exec() cancels the task's uring_cmds before it returns, so the cancel
* is done once the child is reaped. A thread exit does not cancel them,
* and closing the ring cancels them later, from io_ring_exit_work().
*/
#include <sched.h>
#include "kublk.h"
#include "../kselftest.h"
#define NR_QUEUES 2
#define DEPTH 4
#define BUF_SIZE (64 << 10)
#define DEV_SECTORS (64 << 11) /* 64MB */
#define NR_READS (DEPTH * 2)
#define SERVE_DELAY_US 200000
#define RACE_LOOPS 50
#define ASYNC_LOOPS 300
/* one control handle per thread, the race modes send commands from two */
static __thread struct ublk_dev *ctrl_dev;
static int cdev_fd = -1;
static int dev_id = -1;
static int nr_queues;
static void *bufs[NR_QUEUES][DEPTH];
static const struct ublksrv_io_desc *iods[NR_QUEUES];
static size_t iods_len;
static struct io_uring_sqe *get_sqe(struct io_uring *ring)
{
struct io_uring_sqe *sqe = io_uring_get_sqe(ring);
if (!sqe) {
fprintf(stderr, "out of sqes\n");
exit(KSFT_FAIL);
}
return sqe;
}
static struct ublk_dev *ctrl(void)
{
if (!ctrl_dev) {
ctrl_dev = ublk_ctrl_init();
if (!ctrl_dev) {
fprintf(stderr, "ublk_ctrl_init failed\n");
exit(KSFT_FAIL);
}
}
ctrl_dev->dev_info.dev_id = dev_id;
return ctrl_dev;
}
static void ctrl_put(void)
{
if (ctrl_dev)
ublk_ctrl_deinit(ctrl_dev);
ctrl_dev = NULL;
}
static int dev_state(void)
{
struct ublk_dev *dev = ctrl();
int ret = ublk_ctrl_get_info(dev);
return ret ? ret : dev->dev_info.state;
}
static int open_cdev(void)
{
size_t max_len = UBLK_MAX_QUEUE_DEPTH * sizeof(struct ublksrv_io_desc);
int pg = getpagesize();
char path[64];
snprintf(path, sizeof(path), "/dev/ublkc%d", dev_id);
for (int i = 0; i < 100 && cdev_fd < 0; i++) {
cdev_fd = open(path, O_RDWR);
if (cdev_fd < 0)
usleep(50000);
}
if (cdev_fd < 0)
return -errno;
/* queue q's descriptors start at q * the size for the max depth */
max_len = (max_len + pg - 1) & ~(size_t)(pg - 1);
iods_len = (DEPTH * sizeof(struct ublksrv_io_desc) + pg - 1) &
~(size_t)(pg - 1);
for (int q = 0; q < nr_queues; q++) {
void *p = mmap(NULL, iods_len, PROT_READ,
MAP_SHARED | MAP_POPULATE, cdev_fd,
UBLKSRV_CMD_BUF_OFFSET + q * max_len);
if (p == MAP_FAILED)
return -errno;
iods[q] = p;
}
return 0;
}
/* the last reference to /dev/ublkcN runs ublk_ch_release() */
static void close_cdev(void)
{
for (int q = 0; q < nr_queues; q++) {
if (iods[q])
munmap((void *)iods[q], iods_len);
iods[q] = NULL;
}
if (cdev_fd >= 0)
close(cdev_fd);
cdev_fd = -1;
}
/* ADD_DEV and SET_PARAMS, without opening /dev/ublkcN */
static int add_dev_noopen(int queues, __u64 flags)
{
struct ublk_dev *dev = ctrl();
struct ublksrv_ctrl_dev_info info = {
.nr_hw_queues = queues,
.queue_depth = DEPTH,
.max_io_buf_bytes = BUF_SIZE,
.dev_id = -1,
.flags = UBLK_F_NO_AUTO_PART_SCAN | flags,
};
struct ublk_params p = {
.types = UBLK_PARAM_TYPE_BASIC,
.basic = {
.logical_bs_shift = 9,
.physical_bs_shift = 12,
.io_opt_shift = 12,
.io_min_shift = 9,
.max_sectors = BUF_SIZE >> 9,
.dev_sectors = DEV_SECTORS,
},
};
int ret;
nr_queues = queues;
dev->dev_info = info;
ret = ublk_ctrl_add_dev(dev);
if (ret)
return ret;
dev_id = dev->dev_info.dev_id;
ret = ublk_ctrl_set_params(ctrl(), &p);
if (ret)
return ret;
for (int q = 0; q < queues; q++)
for (int i = 0; i < DEPTH; i++)
if (!bufs[q][i] &&
posix_memalign(&bufs[q][i], getpagesize(), BUF_SIZE))
return -ENOMEM;
return 0;
}
static int add_dev(int queues, __u64 flags)
{
return add_dev_noopen(queues, flags) ?: open_cdev();
}
static void cleanup(void)
{
if (dev_id < 0)
return;
ublk_ctrl_stop_dev(ctrl());
close_cdev();
ublk_ctrl_del_dev(ctrl());
dev_id = -1;
}
/* race_async_fetch sets IOSQE_ASYNC on the io commands */
static int io_cmd_sqe_flags;
static void queue_io_cmd(struct io_uring *ring, __u32 op, int q, int tag,
int res)
{
struct io_uring_sqe *sqe = get_sqe(ring);
struct ublksrv_io_cmd *cmd = (struct ublksrv_io_cmd *)sqe->cmd;
memset(sqe, 0, sizeof(*sqe));
sqe->fd = cdev_fd;
sqe->opcode = IORING_OP_URING_CMD;
sqe->flags = io_cmd_sqe_flags;
ublk_set_sqe_cmd_op(sqe, op);
cmd->q_id = q;
cmd->tag = tag;
cmd->result = res;
cmd->addr = (__u64)(uintptr_t)bufs[q][tag];
io_uring_sqe_set_data64(sqe, (q << 16) | tag);
}
struct async_arg {
pthread_barrier_t go, stopped;
};
/*
* FETCH every tag with IOSQE_ASYNC, racing STOP_DEV, then close the ring once
* STOP_DEV returned: that walks io_uring's list of cancelable commands.
*/
static void *race_async_fn(void *data)
{
struct async_arg *a = data;
struct io_uring ring;
int ok = !io_uring_queue_init(DEPTH, &ring, 0);
if (ok)
for (int tag = 0; tag < DEPTH; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, 0, tag, 0);
pthread_barrier_wait(&a->go);
if (ok)
io_uring_submit(&ring);
pthread_barrier_wait(&a->stopped);
if (ok)
io_uring_queue_exit(&ring);
return NULL;
}
/* reap @nr completions of canceled fetch commands, return how many */
static int reap_aborts(struct io_uring *ring, int nr)
{
struct __kernel_timespec ts = { .tv_sec = 2 };
struct io_uring_cqe *cqe;
int aborted = 0;
while (nr--) {
if (io_uring_wait_cqe_timeout(ring, &cqe, &ts))
break;
if (cqe->res == UBLK_IO_RES_ABORT)
aborted++;
io_uring_cqe_seen(ring, cqe);
}
return aborted;
}
/*
* A server thread fetches @nr_tags tags of queue @q, then completes each
* request after @delay_us, until its commands are aborted.
*/
struct server {
int q, first_tag, nr_tags, delay_us;
int failed; /* result which ended the serve loop, 0 if none */
pthread_t thread;
pthread_barrier_t fetched;
/* race_fetch: wait for @go and @race_delay_us, open, fetch one by one */
pthread_barrier_t go;
int race, race_delay_us;
};
static void *server_fn(void *data)
{
struct server *s = data;
struct io_uring ring;
struct io_uring_cqe *cqe;
if (io_uring_queue_init(DEPTH, &ring, 0)) {
pthread_barrier_wait(&s->fetched);
return NULL;
}
if (s->race) {
pthread_barrier_wait(&s->go);
usleep(s->race_delay_us);
if (open_cdev()) {
pthread_barrier_wait(&s->fetched);
io_uring_queue_exit(&ring);
return NULL;
}
}
for (int tag = s->first_tag; tag < s->first_tag + s->nr_tags; tag++) {
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, s->q, tag, 0);
if (s->race)
io_uring_submit(&ring);
}
io_uring_submit(&ring);
pthread_barrier_wait(&s->fetched);
while (!io_uring_wait_cqe(&ring, &cqe)) {
int tag = cqe->user_data & 0xffff;
const struct ublksrv_io_desc *iod = &iods[s->q][tag];
int res = cqe->res;
io_uring_cqe_seen(&ring, cqe);
if (res != UBLK_IO_RES_OK) {
__atomic_store_n(&s->failed, res, __ATOMIC_RELEASE);
break;
}
usleep(s->delay_us);
res = ublksrv_get_op(iod) <= UBLK_IO_OP_WRITE ?
iod->nr_sectors << 9 : 0;
queue_io_cmd(&ring, UBLK_U_IO_COMMIT_AND_FETCH_REQ, s->q, tag,
res);
io_uring_submit(&ring);
}
io_uring_queue_exit(&ring);
return NULL;
}
/* returns once the fetch commands are issued; with @race, call race_go() */
static void server_start(struct server *s)
{
pthread_barrier_init(&s->fetched, NULL, 2);
pthread_barrier_init(&s->go, NULL, 2);
pthread_create(&s->thread, NULL, server_fn, s);
if (!s->race)
pthread_barrier_wait(&s->fetched);
}
/* wait up to 5s for the serve loop of @s to end, return its result */
static int server_wait_failed(struct server *s)
{
int res = 0;
for (int i = 0; i < 500 && !res; i++) {
res = __atomic_load_n(&s->failed, __ATOMIC_ACQUIRE);
if (!res)
usleep(10000);
}
return res;
}
/*
* A task fetches @nr_tags tags of queue @q and dies. Returns once its
* commands are canceled, see the top of this file.
*/
static int fetch_and_die(int q, int first_tag, int nr_tags)
{
int status;
pid_t pid = fork();
if (pid < 0)
return -1;
if (!pid) {
struct io_uring ring;
if (io_uring_queue_init(DEPTH, &ring, 0))
_exit(1);
for (int tag = first_tag; tag < first_tag + nr_tags; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, q, tag, 0);
if (io_uring_submit(&ring) != nr_tags)
_exit(1);
execlp("true", "true", NULL);
_exit(1);
}
if (waitpid(pid, &status, 0) != pid || !WIFEXITED(status) ||
WEXITSTATUS(status))
return -1;
printf("q%d: task died with %d fetch cmds in flight\n", q, nr_tags);
return 0;
}
struct reads {
struct io_uring ring;
void *buf;
int fd, done, ok, eio, other;
};
/*
* Issue NR_READS reads at once, from @cpu if it is not negative: blk-mq
* maps the submitting CPU to the hw queue. A server holding its live
* tags for a while makes the other reads take the canceled tags.
*/
static int open_tries = 100; /* 50ms each */
static int reads_submit(struct reads *r, int cpu)
{
char path[64];
cpu_set_t set, old;
snprintf(path, sizeof(path), "/dev/ublkb%d", dev_id);
r->fd = -1;
for (int i = 0; i < open_tries && r->fd < 0; i++) {
r->fd = open(path, O_RDONLY | O_DIRECT);
if (r->fd < 0)
usleep(50000);
}
if (r->fd < 0) {
fprintf(stderr, "open %s: %s\n", path, strerror(errno));
return -1;
}
if (posix_memalign(&r->buf, 4096, NR_READS * 4096) ||
io_uring_queue_init(NR_READS, &r->ring, 0))
return -1;
for (int i = 0; i < NR_READS; i++)
io_uring_prep_read(get_sqe(&r->ring), r->fd,
r->buf + i * 4096, 4096, i * 4096);
if (cpu >= 0) {
sched_getaffinity(0, sizeof(old), &old);
CPU_ZERO(&set);
CPU_SET(cpu, &set);
sched_setaffinity(0, sizeof(set), &set);
}
io_uring_submit(&r->ring);
if (cpu >= 0)
sched_setaffinity(0, sizeof(old), &old);
return 0;
}
static void reads_reap(struct reads *r, int timeout_s)
{
struct __kernel_timespec ts = { .tv_sec = timeout_s };
struct io_uring_cqe *cqe;
while (r->done < NR_READS &&
!io_uring_wait_cqe_timeout(&r->ring, &cqe, &ts)) {
if (cqe->res == 4096)
r->ok++;
else if (cqe->res == -EIO)
r->eio++;
else
r->other++;
r->done++;
io_uring_cqe_seen(&r->ring, cqe);
}
}
static void reads_put(struct reads *r)
{
io_uring_queue_exit(&r->ring);
close(r->fd);
free(r->buf);
}
static int reads_result(struct reads *r)
{
printf("reads: %d ok, %d -EIO, %d other, %d not completed\n",
r->ok, r->eio, r->other, NR_READS - r->done);
reads_put(r);
return r->other || r->done < NR_READS ? KSFT_FAIL : KSFT_PASS;
}
static int start_dev(void)
{
int ret = ublk_ctrl_start_dev(ctrl(), getpid());
printf("START_DEV: %d\n", ret);
return ret;
}
static int expect_err(const char *what, int ret, int want)
{
if (ret == want)
return KSFT_PASS;
fprintf(stderr, "%s: %d, expected %d\n", what, ret, want);
return KSFT_FAIL;
}
/* START_DEV must fail with @want; if it went live, show what a read does */
static int start_dev_expect(int want)
{
struct reads r = {};
int ret = start_dev();
if (!ret && !reads_submit(&r, -1)) {
reads_reap(&r, 10);
reads_result(&r);
}
return expect_err("START_DEV", ret, want);
}
static int test_stop_start(void)
{
struct io_uring ring;
int ret;
if (add_dev(1, 0))
return KSFT_FAIL;
if (io_uring_queue_init(DEPTH, &ring, 0))
return KSFT_FAIL;
for (int tag = 0; tag < DEPTH; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, 0, tag, 0);
io_uring_submit(&ring);
/* device is ready but not started: state is UBLK_S_DEV_DEAD */
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d, canceled fetch cmds: %d/%d\n", ret,
reap_aborts(&ring, DEPTH), DEPTH);
/* STOP_DEV canceled the attached server: no start until it exits */
ret = start_dev_expect(-EBUSY);
io_uring_queue_exit(&ring);
return ret;
}
static int test_partial_fetch(void)
{
struct server b = { .q = 0, .first_tag = DEPTH - 1, .nr_tags = 1,
.delay_us = SERVE_DELAY_US };
int ret;
if (add_dev(1, 0) || fetch_and_die(0, 0, DEPTH - 1))
return KSFT_FAIL;
/* the last FETCH makes the queue ready and clears ->canceling */
server_start(&b);
ret = start_dev_expect(-EBUSY);
cleanup();
pthread_join(b.thread, NULL);
return ret;
}
static int test_stop_restart(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
struct reads r = {};
int ret;
if (add_dev_noopen(1, 0))
return KSFT_FAIL;
/* no server is attached, so there is nothing to cancel */
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d\n", ret);
if (open_cdev())
return KSFT_FAIL;
server_start(&s);
if (start_dev()) {
ret = KSFT_FAIL;
} else if (reads_submit(&r, -1)) {
ret = KSFT_FAIL;
} else {
reads_reap(&r, 10);
ret = reads_result(&r);
if (r.ok != NR_READS)
ret = KSFT_FAIL;
}
cleanup();
pthread_join(s.thread, NULL);
return ret;
}
struct start_arg {
pthread_barrier_t go;
int ret, reads_ok;
};
static void *race_start_fn(void *data)
{
struct start_arg *a = data;
struct reads r = {};
pthread_barrier_wait(&a->go);
a->ret = ublk_ctrl_start_dev(ctrl(), getpid());
a->reads_ok = 1;
/*
* The disk may be gone already if STOP_DEV came right after, and
* reads may fail then: they only have to complete.
*/
if (!a->ret && !reads_submit(&r, -1)) {
reads_reap(&r, 5);
a->reads_ok = r.done == NR_READS;
if (a->reads_ok)
reads_put(&r);
else
reads_result(&r);
}
ctrl_put();
return NULL;
}
static int test_race_start(void)
{
int live = 0, ebusy = 0;
open_tries = 4;
for (int i = 0; i < RACE_LOOPS; i++) {
struct server s = { .q = 0, .nr_tags = DEPTH };
struct start_arg a = {};
pthread_t t;
if (add_dev(1, 0))
return KSFT_FAIL;
server_start(&s);
pthread_barrier_init(&a.go, NULL, 2);
pthread_create(&t, NULL, race_start_fn, &a);
pthread_barrier_wait(&a.go);
ublk_ctrl_stop_dev(ctrl());
pthread_join(t, NULL);
cleanup();
pthread_join(s.thread, NULL);
if (a.ret == 0)
live++;
else if (a.ret == -EBUSY)
ebusy++;
if ((a.ret && a.ret != -EBUSY) || !a.reads_ok) {
fprintf(stderr, "loop %d: START_DEV %d, reads %s\n",
i, a.ret, a.reads_ok ? "ok" : "failed");
return KSFT_FAIL;
}
}
printf("%d loops: START_DEV won %d, got -EBUSY %d\n", RACE_LOOPS,
live, ebusy);
return KSFT_PASS;
}
static int test_race_fetch(void)
{
int live = 0, ebusy = 0;
for (int i = 0; i < RACE_LOOPS; i++) {
/* vary who goes first: STOP_DEV, or the server's open */
struct server s = { .q = 0, .nr_tags = DEPTH, .race = 1,
.race_delay_us = (i % 10) * 50 };
struct reads r = {};
int ret;
if (add_dev_noopen(1, 0))
return KSFT_FAIL;
server_start(&s);
pthread_barrier_wait(&s.go);
ublk_ctrl_stop_dev(ctrl());
pthread_barrier_wait(&s.fetched);
ret = ublk_ctrl_start_dev(ctrl(), getpid());
if (!ret) {
/* nothing was taken, so every read has to succeed */
live++;
if (reads_submit(&r, -1))
return KSFT_FAIL;
reads_reap(&r, 5);
if (r.ok != NR_READS) {
fprintf(stderr, "loop %d: live, but ", i);
reads_result(&r);
return KSFT_FAIL;
}
reads_put(&r);
} else if (ret == -EBUSY) {
ebusy++;
} else {
fprintf(stderr, "loop %d: START_DEV %d\n", i, ret);
return KSFT_FAIL;
}
cleanup();
pthread_join(s.thread, NULL);
}
printf("%d loops: START_DEV worked %d, got -EBUSY %d\n", RACE_LOOPS,
live, ebusy);
return KSFT_PASS;
}
static int test_race_async_fetch(void)
{
io_cmd_sqe_flags = IOSQE_ASYNC;
for (int i = 0; i < ASYNC_LOOPS; i++) {
struct async_arg a;
pthread_t t;
if (add_dev(1, 0))
return KSFT_FAIL;
pthread_barrier_init(&a.go, NULL, 2);
pthread_barrier_init(&a.stopped, NULL, 2);
pthread_create(&t, NULL, race_async_fn, &a);
pthread_barrier_wait(&a.go);
/* vary where STOP_DEV lands relative to the FETCHes */
usleep((i % 5) * 1000);
ublk_ctrl_stop_dev(ctrl());
pthread_barrier_wait(&a.stopped);
pthread_join(t, NULL);
cleanup();
}
printf("%d loops done\n", ASYNC_LOOPS);
return KSFT_PASS;
}
/*
* The stopped server is gone: reopen /dev/ublkcN, start a new server and
* the device, and read from it.
*/
static int restart_server(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
struct reads r = {};
int ret;
close_cdev();
if (open_cdev())
return KSFT_FAIL;
server_start(&s);
usleep(200000);
if (s.failed) {
fprintf(stderr, "new server: FETCH failed %d\n", s.failed);
cleanup();
pthread_join(s.thread, NULL);
return KSFT_FAIL;
}
/* -EEXIST until the old disk is freed, e.g. after a udev probe */
for (int i = 0; i < 100; i++) {
ret = start_dev();
if (ret != -EEXIST)
break;
usleep(50000);
}
if (ret || reads_submit(&r, -1)) {
fprintf(stderr, "new server: START_DEV %d\n", ret);
ret = KSFT_FAIL;
} else {
reads_reap(&r, 10);
ret = reads_result(&r);
if (r.ok != NR_READS)
ret = KSFT_FAIL;
}
cleanup();
pthread_join(s.thread, NULL);
return ret;
}
/*
* stop_attached: STOP_DEV while a server has the device open but has
* fetched nothing stops that server too: its FETCH gets ABORT and
* START_DEV -EBUSY. Once it is gone, a new server can start the device.
*/
static int test_stop_attached(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
int ret;
if (add_dev(1, 0))
return KSFT_FAIL;
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d\n", ret);
server_start(&s);
ret = server_wait_failed(&s);
printf("FETCH after STOP_DEV: %d\n", ret);
if (ret != UBLK_IO_RES_ABORT) {
cleanup();
pthread_join(s.thread, NULL);
return KSFT_FAIL;
}
pthread_join(s.thread, NULL);
if (expect_err("START_DEV", start_dev(), -EBUSY))
return KSFT_FAIL;
return restart_server();
}
/*
* stop_live_restart: STOP_DEV on a live device; once its server is gone,
* a new server has to fetch and start it again.
*/
static int test_stop_live_restart(void)
{
struct server s = { .q = 0, .nr_tags = DEPTH };
int ret;
if (add_dev(1, 0))
return KSFT_FAIL;
server_start(&s);
if (expect_err("START_DEV", start_dev(), 0))
return KSFT_FAIL;
ret = ublk_ctrl_stop_dev(ctrl());
printf("STOP_DEV: %d\n", ret);
pthread_join(s.thread, NULL);
return restart_server();
}
/* first CPU which blk-mq maps to hw queue @q */
static int queue_cpu(int q)
{
char path[96];
FILE *f;
int cpu = -1;
snprintf(path, sizeof(path), "/sys/block/ublkb%d/mq/%d/cpu_list",
dev_id, q);
for (int i = 0; i < 100 && !(f = fopen(path, "r")); i++)
usleep(50000);
if (!f)
return -1;
if (fscanf(f, "%d", &cpu) != 1)
cpu = -1;
fclose(f);
return cpu;
}
static int test_recovery(void)
{
struct server q1 = { .q = 1, .nr_tags = DEPTH };
struct io_uring ring;
struct reads r = {};
int ret, cpu;
if (add_dev(NR_QUEUES, UBLK_F_USER_RECOVERY))
return KSFT_FAIL;
/* the first server: fetch everything, start, then die */
if (io_uring_queue_init(NR_QUEUES * DEPTH, &ring, 0))
return KSFT_FAIL;
for (int q = 0; q < NR_QUEUES; q++)
for (int tag = 0; tag < DEPTH; tag++)
queue_io_cmd(&ring, UBLK_U_IO_FETCH_REQ, q, tag, 0);
io_uring_submit(&ring);
if (start_dev())
return KSFT_FAIL;
cpu = queue_cpu(0);
if (cpu < 0) {
printf("no CPU maps to queue 0, can't aim the reads\n");
return KSFT_SKIP;
}
io_uring_queue_exit(&ring);
close_cdev();
for (int i = 0; i < 100 && dev_state() != UBLK_S_DEV_QUIESCED; i++)
usleep(50000);
printf("server exited, state %d (QUIESCED is %d)\n", dev_state(),
UBLK_S_DEV_QUIESCED);
for (int i = 0; i < 100; i++) {
ret = ublk_ctrl_start_user_recovery(ctrl());
if (ret != -EBUSY)
break;
usleep(50000);
}
printf("START_USER_RECOVERY: %d\n", ret);
if (ret || open_cdev())
return KSFT_FAIL;
/* queue 0 gets ready, then its task dies before queue 1 is ready */
if (fetch_and_die(0, 0, DEPTH))
return KSFT_FAIL;
printf("reads on cpu %d, which maps to queue 0\n", cpu);
if (reads_submit(&r, cpu))
return KSFT_FAIL;
server_start(&q1);
ret = ublk_ctrl_end_user_recovery(ctrl(), getpid());
printf("END_USER_RECOVERY: %d\n", ret);
ret = expect_err("END_USER_RECOVERY", ret, -EBUSY);
/*
* Requests held back on queue 0 wait for STOP_DEV to fail them.
* Close the disk before DEL_DEV, which waits for its last reference.
*/
reads_reap(&r, 1);
ublk_ctrl_stop_dev(ctrl());
reads_reap(&r, 5);
if (reads_result(&r))
ret = KSFT_FAIL;
cleanup();
pthread_join(q1.thread, NULL);
return ret;
}
static const struct {
const char *name;
int (*fn)(void);
} modes[] = {
{ "stop_start", test_stop_start },
{ "partial_fetch", test_partial_fetch },
{ "recovery", test_recovery },
{ "stop_restart", test_stop_restart },
{ "race_start", test_race_start },
{ "race_fetch", test_race_fetch },
{ "race_async_fetch", test_race_async_fetch },
{ "stop_attached", test_stop_attached },
{ "stop_live_restart", test_stop_live_restart },
};
int main(int argc, char **argv)
{
int (*fn)(void) = NULL;
int ret;
for (int i = 0; argc == 2 && i < ARRAY_SIZE(modes); i++)
if (!strcmp(argv[1], modes[i].name))
fn = modes[i].fn;
if (!fn) {
fprintf(stderr, "usage: %s MODE, modes:", argv[0]);
for (int i = 0; i < ARRAY_SIZE(modes); i++)
fprintf(stderr, " %s", modes[i].name);
fprintf(stderr, "\n");
return KSFT_FAIL;
}
/* keep the output of a run that ends in an oops */
setvbuf(stdout, NULL, _IOLBF, 0);
if (access(CTRL_DEV, F_OK)) {
perror(CTRL_DEV);
return KSFT_SKIP;
}
printf("%s\n", argv[1]);
ret = fn();
cleanup();
ctrl_put();
return ret;
}