arm-semi: don't leak 1KB user string lock buffer upon TARGET_SYS_OPEN
[qemu-kvm.git] / block / sheepdog.c
blobdf4f44107b32f540d26ae3e4763ee440841ef7cf
1 /*
2 * Copyright (C) 2009-2010 Nippon Telegraph and Telephone Corporation.
4 * This program is free software; you can redistribute it and/or
5 * modify it under the terms of the GNU General Public License version
6 * 2 as published by the Free Software Foundation.
8 * You should have received a copy of the GNU General Public License
9 * along with this program. If not, see <http://www.gnu.org/licenses/>.
11 * Contributions after 2012-01-13 are licensed under the terms of the
12 * GNU GPL, version 2 or (at your option) any later version.
15 #include "qemu-common.h"
16 #include "qemu-error.h"
17 #include "qemu_socket.h"
18 #include "block_int.h"
19 #include "bitops.h"
21 #define SD_PROTO_VER 0x01
23 #define SD_DEFAULT_ADDR "localhost"
24 #define SD_DEFAULT_PORT "7000"
26 #define SD_OP_CREATE_AND_WRITE_OBJ 0x01
27 #define SD_OP_READ_OBJ 0x02
28 #define SD_OP_WRITE_OBJ 0x03
30 #define SD_OP_NEW_VDI 0x11
31 #define SD_OP_LOCK_VDI 0x12
32 #define SD_OP_RELEASE_VDI 0x13
33 #define SD_OP_GET_VDI_INFO 0x14
34 #define SD_OP_READ_VDIS 0x15
35 #define SD_OP_FLUSH_VDI 0x16
37 #define SD_FLAG_CMD_WRITE 0x01
38 #define SD_FLAG_CMD_COW 0x02
39 #define SD_FLAG_CMD_CACHE 0x04
41 #define SD_RES_SUCCESS 0x00 /* Success */
42 #define SD_RES_UNKNOWN 0x01 /* Unknown error */
43 #define SD_RES_NO_OBJ 0x02 /* No object found */
44 #define SD_RES_EIO 0x03 /* I/O error */
45 #define SD_RES_VDI_EXIST 0x04 /* Vdi exists already */
46 #define SD_RES_INVALID_PARMS 0x05 /* Invalid parameters */
47 #define SD_RES_SYSTEM_ERROR 0x06 /* System error */
48 #define SD_RES_VDI_LOCKED 0x07 /* Vdi is locked */
49 #define SD_RES_NO_VDI 0x08 /* No vdi found */
50 #define SD_RES_NO_BASE_VDI 0x09 /* No base vdi found */
51 #define SD_RES_VDI_READ 0x0A /* Cannot read requested vdi */
52 #define SD_RES_VDI_WRITE 0x0B /* Cannot write requested vdi */
53 #define SD_RES_BASE_VDI_READ 0x0C /* Cannot read base vdi */
54 #define SD_RES_BASE_VDI_WRITE 0x0D /* Cannot write base vdi */
55 #define SD_RES_NO_TAG 0x0E /* Requested tag is not found */
56 #define SD_RES_STARTUP 0x0F /* Sheepdog is on starting up */
57 #define SD_RES_VDI_NOT_LOCKED 0x10 /* Vdi is not locked */
58 #define SD_RES_SHUTDOWN 0x11 /* Sheepdog is shutting down */
59 #define SD_RES_NO_MEM 0x12 /* Cannot allocate memory */
60 #define SD_RES_FULL_VDI 0x13 /* we already have the maximum vdis */
61 #define SD_RES_VER_MISMATCH 0x14 /* Protocol version mismatch */
62 #define SD_RES_NO_SPACE 0x15 /* Server has no room for new objects */
63 #define SD_RES_WAIT_FOR_FORMAT 0x16 /* Waiting for a format operation */
64 #define SD_RES_WAIT_FOR_JOIN 0x17 /* Waiting for other nodes joining */
65 #define SD_RES_JOIN_FAILED 0x18 /* Target node had failed to join sheepdog */
68 * Object ID rules
70 * 0 - 19 (20 bits): data object space
71 * 20 - 31 (12 bits): reserved data object space
72 * 32 - 55 (24 bits): vdi object space
73 * 56 - 59 ( 4 bits): reserved vdi object space
74 * 60 - 63 ( 4 bits): object type identifier space
77 #define VDI_SPACE_SHIFT 32
78 #define VDI_BIT (UINT64_C(1) << 63)
79 #define VMSTATE_BIT (UINT64_C(1) << 62)
80 #define MAX_DATA_OBJS (UINT64_C(1) << 20)
81 #define MAX_CHILDREN 1024
82 #define SD_MAX_VDI_LEN 256
83 #define SD_MAX_VDI_TAG_LEN 256
84 #define SD_NR_VDIS (1U << 24)
85 #define SD_DATA_OBJ_SIZE (UINT64_C(1) << 22)
86 #define SD_MAX_VDI_SIZE (SD_DATA_OBJ_SIZE * MAX_DATA_OBJS)
87 #define SECTOR_SIZE 512
89 #define SD_INODE_SIZE (sizeof(SheepdogInode))
90 #define CURRENT_VDI_ID 0
92 typedef struct SheepdogReq {
93 uint8_t proto_ver;
94 uint8_t opcode;
95 uint16_t flags;
96 uint32_t epoch;
97 uint32_t id;
98 uint32_t data_length;
99 uint32_t opcode_specific[8];
100 } SheepdogReq;
102 typedef struct SheepdogRsp {
103 uint8_t proto_ver;
104 uint8_t opcode;
105 uint16_t flags;
106 uint32_t epoch;
107 uint32_t id;
108 uint32_t data_length;
109 uint32_t result;
110 uint32_t opcode_specific[7];
111 } SheepdogRsp;
113 typedef struct SheepdogObjReq {
114 uint8_t proto_ver;
115 uint8_t opcode;
116 uint16_t flags;
117 uint32_t epoch;
118 uint32_t id;
119 uint32_t data_length;
120 uint64_t oid;
121 uint64_t cow_oid;
122 uint32_t copies;
123 uint32_t rsvd;
124 uint64_t offset;
125 } SheepdogObjReq;
127 typedef struct SheepdogObjRsp {
128 uint8_t proto_ver;
129 uint8_t opcode;
130 uint16_t flags;
131 uint32_t epoch;
132 uint32_t id;
133 uint32_t data_length;
134 uint32_t result;
135 uint32_t copies;
136 uint32_t pad[6];
137 } SheepdogObjRsp;
139 typedef struct SheepdogVdiReq {
140 uint8_t proto_ver;
141 uint8_t opcode;
142 uint16_t flags;
143 uint32_t epoch;
144 uint32_t id;
145 uint32_t data_length;
146 uint64_t vdi_size;
147 uint32_t base_vdi_id;
148 uint32_t copies;
149 uint32_t snapid;
150 uint32_t pad[3];
151 } SheepdogVdiReq;
153 typedef struct SheepdogVdiRsp {
154 uint8_t proto_ver;
155 uint8_t opcode;
156 uint16_t flags;
157 uint32_t epoch;
158 uint32_t id;
159 uint32_t data_length;
160 uint32_t result;
161 uint32_t rsvd;
162 uint32_t vdi_id;
163 uint32_t pad[5];
164 } SheepdogVdiRsp;
166 typedef struct SheepdogInode {
167 char name[SD_MAX_VDI_LEN];
168 char tag[SD_MAX_VDI_TAG_LEN];
169 uint64_t ctime;
170 uint64_t snap_ctime;
171 uint64_t vm_clock_nsec;
172 uint64_t vdi_size;
173 uint64_t vm_state_size;
174 uint16_t copy_policy;
175 uint8_t nr_copies;
176 uint8_t block_size_shift;
177 uint32_t snap_id;
178 uint32_t vdi_id;
179 uint32_t parent_vdi_id;
180 uint32_t child_vdi_id[MAX_CHILDREN];
181 uint32_t data_vdi_id[MAX_DATA_OBJS];
182 } SheepdogInode;
185 * 64 bit FNV-1a non-zero initial basis
187 #define FNV1A_64_INIT ((uint64_t)0xcbf29ce484222325ULL)
190 * 64 bit Fowler/Noll/Vo FNV-1a hash code
192 static inline uint64_t fnv_64a_buf(void *buf, size_t len, uint64_t hval)
194 unsigned char *bp = buf;
195 unsigned char *be = bp + len;
196 while (bp < be) {
197 hval ^= (uint64_t) *bp++;
198 hval += (hval << 1) + (hval << 4) + (hval << 5) +
199 (hval << 7) + (hval << 8) + (hval << 40);
201 return hval;
204 static inline int is_data_obj_writable(SheepdogInode *inode, unsigned int idx)
206 return inode->vdi_id == inode->data_vdi_id[idx];
209 static inline int is_data_obj(uint64_t oid)
211 return !(VDI_BIT & oid);
214 static inline uint64_t data_oid_to_idx(uint64_t oid)
216 return oid & (MAX_DATA_OBJS - 1);
219 static inline uint64_t vid_to_vdi_oid(uint32_t vid)
221 return VDI_BIT | ((uint64_t)vid << VDI_SPACE_SHIFT);
224 static inline uint64_t vid_to_vmstate_oid(uint32_t vid, uint32_t idx)
226 return VMSTATE_BIT | ((uint64_t)vid << VDI_SPACE_SHIFT) | idx;
229 static inline uint64_t vid_to_data_oid(uint32_t vid, uint32_t idx)
231 return ((uint64_t)vid << VDI_SPACE_SHIFT) | idx;
234 static inline int is_snapshot(struct SheepdogInode *inode)
236 return !!inode->snap_ctime;
239 #undef dprintf
240 #ifdef DEBUG_SDOG
241 #define dprintf(fmt, args...) \
242 do { \
243 fprintf(stdout, "%s %d: " fmt, __func__, __LINE__, ##args); \
244 } while (0)
245 #else
246 #define dprintf(fmt, args...)
247 #endif
249 typedef struct SheepdogAIOCB SheepdogAIOCB;
251 typedef struct AIOReq {
252 SheepdogAIOCB *aiocb;
253 unsigned int iov_offset;
255 uint64_t oid;
256 uint64_t base_oid;
257 uint64_t offset;
258 unsigned int data_len;
259 uint8_t flags;
260 uint32_t id;
262 QLIST_ENTRY(AIOReq) aio_siblings;
263 } AIOReq;
265 enum AIOCBState {
266 AIOCB_WRITE_UDATA,
267 AIOCB_READ_UDATA,
270 struct SheepdogAIOCB {
271 BlockDriverAIOCB common;
273 QEMUIOVector *qiov;
275 int64_t sector_num;
276 int nb_sectors;
278 int ret;
279 enum AIOCBState aiocb_type;
281 Coroutine *coroutine;
282 void (*aio_done_func)(SheepdogAIOCB *);
284 int canceled;
285 int nr_pending;
288 typedef struct BDRVSheepdogState {
289 SheepdogInode inode;
291 uint32_t min_dirty_data_idx;
292 uint32_t max_dirty_data_idx;
294 char name[SD_MAX_VDI_LEN];
295 int is_snapshot;
296 uint8_t cache_enabled;
298 char *addr;
299 char *port;
300 int fd;
301 int flush_fd;
303 CoMutex lock;
304 Coroutine *co_send;
305 Coroutine *co_recv;
307 uint32_t aioreq_seq_num;
308 QLIST_HEAD(inflight_aio_head, AIOReq) inflight_aio_head;
309 QLIST_HEAD(pending_aio_head, AIOReq) pending_aio_head;
310 } BDRVSheepdogState;
312 static const char * sd_strerror(int err)
314 int i;
316 static const struct {
317 int err;
318 const char *desc;
319 } errors[] = {
320 {SD_RES_SUCCESS, "Success"},
321 {SD_RES_UNKNOWN, "Unknown error"},
322 {SD_RES_NO_OBJ, "No object found"},
323 {SD_RES_EIO, "I/O error"},
324 {SD_RES_VDI_EXIST, "VDI exists already"},
325 {SD_RES_INVALID_PARMS, "Invalid parameters"},
326 {SD_RES_SYSTEM_ERROR, "System error"},
327 {SD_RES_VDI_LOCKED, "VDI is already locked"},
328 {SD_RES_NO_VDI, "No vdi found"},
329 {SD_RES_NO_BASE_VDI, "No base VDI found"},
330 {SD_RES_VDI_READ, "Failed read the requested VDI"},
331 {SD_RES_VDI_WRITE, "Failed to write the requested VDI"},
332 {SD_RES_BASE_VDI_READ, "Failed to read the base VDI"},
333 {SD_RES_BASE_VDI_WRITE, "Failed to write the base VDI"},
334 {SD_RES_NO_TAG, "Failed to find the requested tag"},
335 {SD_RES_STARTUP, "The system is still booting"},
336 {SD_RES_VDI_NOT_LOCKED, "VDI isn't locked"},
337 {SD_RES_SHUTDOWN, "The system is shutting down"},
338 {SD_RES_NO_MEM, "Out of memory on the server"},
339 {SD_RES_FULL_VDI, "We already have the maximum vdis"},
340 {SD_RES_VER_MISMATCH, "Protocol version mismatch"},
341 {SD_RES_NO_SPACE, "Server has no space for new objects"},
342 {SD_RES_WAIT_FOR_FORMAT, "Sheepdog is waiting for a format operation"},
343 {SD_RES_WAIT_FOR_JOIN, "Sheepdog is waiting for other nodes joining"},
344 {SD_RES_JOIN_FAILED, "Target node had failed to join sheepdog"},
347 for (i = 0; i < ARRAY_SIZE(errors); ++i) {
348 if (errors[i].err == err) {
349 return errors[i].desc;
353 return "Invalid error code";
357 * Sheepdog I/O handling:
359 * 1. In sd_co_rw_vector, we send the I/O requests to the server and
360 * link the requests to the inflight_list in the
361 * BDRVSheepdogState. The function exits without waiting for
362 * receiving the response.
364 * 2. We receive the response in aio_read_response, the fd handler to
365 * the sheepdog connection. If metadata update is needed, we send
366 * the write request to the vdi object in sd_write_done, the write
367 * completion function. We switch back to sd_co_readv/writev after
368 * all the requests belonging to the AIOCB are finished.
371 static inline AIOReq *alloc_aio_req(BDRVSheepdogState *s, SheepdogAIOCB *acb,
372 uint64_t oid, unsigned int data_len,
373 uint64_t offset, uint8_t flags,
374 uint64_t base_oid, unsigned int iov_offset)
376 AIOReq *aio_req;
378 aio_req = g_malloc(sizeof(*aio_req));
379 aio_req->aiocb = acb;
380 aio_req->iov_offset = iov_offset;
381 aio_req->oid = oid;
382 aio_req->base_oid = base_oid;
383 aio_req->offset = offset;
384 aio_req->data_len = data_len;
385 aio_req->flags = flags;
386 aio_req->id = s->aioreq_seq_num++;
388 acb->nr_pending++;
389 return aio_req;
392 static inline void free_aio_req(BDRVSheepdogState *s, AIOReq *aio_req)
394 SheepdogAIOCB *acb = aio_req->aiocb;
396 QLIST_REMOVE(aio_req, aio_siblings);
397 g_free(aio_req);
399 acb->nr_pending--;
402 static void coroutine_fn sd_finish_aiocb(SheepdogAIOCB *acb)
404 if (!acb->canceled) {
405 qemu_coroutine_enter(acb->coroutine, NULL);
407 qemu_aio_release(acb);
410 static void sd_aio_cancel(BlockDriverAIOCB *blockacb)
412 SheepdogAIOCB *acb = (SheepdogAIOCB *)blockacb;
415 * Sheepdog cannot cancel the requests which are already sent to
416 * the servers, so we just complete the request with -EIO here.
418 acb->ret = -EIO;
419 qemu_coroutine_enter(acb->coroutine, NULL);
420 acb->canceled = 1;
423 static AIOPool sd_aio_pool = {
424 .aiocb_size = sizeof(SheepdogAIOCB),
425 .cancel = sd_aio_cancel,
428 static SheepdogAIOCB *sd_aio_setup(BlockDriverState *bs, QEMUIOVector *qiov,
429 int64_t sector_num, int nb_sectors,
430 BlockDriverCompletionFunc *cb, void *opaque)
432 SheepdogAIOCB *acb;
434 acb = qemu_aio_get(&sd_aio_pool, bs, cb, opaque);
436 acb->qiov = qiov;
438 acb->sector_num = sector_num;
439 acb->nb_sectors = nb_sectors;
441 acb->aio_done_func = NULL;
442 acb->canceled = 0;
443 acb->coroutine = qemu_coroutine_self();
444 acb->ret = 0;
445 acb->nr_pending = 0;
446 return acb;
449 static int connect_to_sdog(const char *addr, const char *port)
451 char hbuf[NI_MAXHOST], sbuf[NI_MAXSERV];
452 int fd, ret;
453 struct addrinfo hints, *res, *res0;
455 if (!addr) {
456 addr = SD_DEFAULT_ADDR;
457 port = SD_DEFAULT_PORT;
460 memset(&hints, 0, sizeof(hints));
461 hints.ai_socktype = SOCK_STREAM;
463 ret = getaddrinfo(addr, port, &hints, &res0);
464 if (ret) {
465 error_report("unable to get address info %s, %s",
466 addr, strerror(errno));
467 return -errno;
470 for (res = res0; res; res = res->ai_next) {
471 ret = getnameinfo(res->ai_addr, res->ai_addrlen, hbuf, sizeof(hbuf),
472 sbuf, sizeof(sbuf), NI_NUMERICHOST | NI_NUMERICSERV);
473 if (ret) {
474 continue;
477 fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol);
478 if (fd < 0) {
479 continue;
482 reconnect:
483 ret = connect(fd, res->ai_addr, res->ai_addrlen);
484 if (ret < 0) {
485 if (errno == EINTR) {
486 goto reconnect;
488 close(fd);
489 break;
492 dprintf("connected to %s:%s\n", addr, port);
493 goto success;
495 fd = -errno;
496 error_report("failed connect to %s:%s", addr, port);
497 success:
498 freeaddrinfo(res0);
499 return fd;
502 static coroutine_fn int send_co_req(int sockfd, SheepdogReq *hdr, void *data,
503 unsigned int *wlen)
505 int ret;
507 ret = qemu_co_send(sockfd, hdr, sizeof(*hdr));
508 if (ret < sizeof(*hdr)) {
509 error_report("failed to send a req, %s", strerror(errno));
510 return ret;
513 ret = qemu_co_send(sockfd, data, *wlen);
514 if (ret < *wlen) {
515 error_report("failed to send a req, %s", strerror(errno));
518 return ret;
521 static void restart_co_req(void *opaque)
523 Coroutine *co = opaque;
525 qemu_coroutine_enter(co, NULL);
528 typedef struct SheepdogReqCo {
529 int sockfd;
530 SheepdogReq *hdr;
531 void *data;
532 unsigned int *wlen;
533 unsigned int *rlen;
534 int ret;
535 bool finished;
536 } SheepdogReqCo;
538 static coroutine_fn void do_co_req(void *opaque)
540 int ret;
541 Coroutine *co;
542 SheepdogReqCo *srco = opaque;
543 int sockfd = srco->sockfd;
544 SheepdogReq *hdr = srco->hdr;
545 void *data = srco->data;
546 unsigned int *wlen = srco->wlen;
547 unsigned int *rlen = srco->rlen;
549 co = qemu_coroutine_self();
550 qemu_aio_set_fd_handler(sockfd, NULL, restart_co_req, NULL, co);
552 socket_set_block(sockfd);
553 ret = send_co_req(sockfd, hdr, data, wlen);
554 if (ret < 0) {
555 goto out;
558 qemu_aio_set_fd_handler(sockfd, restart_co_req, NULL, NULL, co);
560 ret = qemu_co_recv(sockfd, hdr, sizeof(*hdr));
561 if (ret < sizeof(*hdr)) {
562 error_report("failed to get a rsp, %s", strerror(errno));
563 ret = -errno;
564 goto out;
567 if (*rlen > hdr->data_length) {
568 *rlen = hdr->data_length;
571 if (*rlen) {
572 ret = qemu_co_recv(sockfd, data, *rlen);
573 if (ret < *rlen) {
574 error_report("failed to get the data, %s", strerror(errno));
575 ret = -errno;
576 goto out;
579 ret = 0;
580 out:
581 qemu_aio_set_fd_handler(sockfd, NULL, NULL, NULL, NULL);
582 socket_set_nonblock(sockfd);
584 srco->ret = ret;
585 srco->finished = true;
588 static int do_req(int sockfd, SheepdogReq *hdr, void *data,
589 unsigned int *wlen, unsigned int *rlen)
591 Coroutine *co;
592 SheepdogReqCo srco = {
593 .sockfd = sockfd,
594 .hdr = hdr,
595 .data = data,
596 .wlen = wlen,
597 .rlen = rlen,
598 .ret = 0,
599 .finished = false,
602 if (qemu_in_coroutine()) {
603 do_co_req(&srco);
604 } else {
605 co = qemu_coroutine_create(do_co_req);
606 qemu_coroutine_enter(co, &srco);
607 while (!srco.finished) {
608 qemu_aio_wait();
612 return srco.ret;
615 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
616 struct iovec *iov, int niov, int create,
617 enum AIOCBState aiocb_type);
620 static AIOReq *find_pending_req(BDRVSheepdogState *s, uint64_t oid)
622 AIOReq *aio_req;
624 QLIST_FOREACH(aio_req, &s->pending_aio_head, aio_siblings) {
625 if (aio_req->oid == oid) {
626 return aio_req;
630 return NULL;
634 * This function searchs pending requests to the object `oid', and
635 * sends them.
637 static void coroutine_fn send_pending_req(BDRVSheepdogState *s, uint64_t oid)
639 AIOReq *aio_req;
640 SheepdogAIOCB *acb;
641 int ret;
643 while ((aio_req = find_pending_req(s, oid)) != NULL) {
644 acb = aio_req->aiocb;
645 /* move aio_req from pending list to inflight one */
646 QLIST_REMOVE(aio_req, aio_siblings);
647 QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
648 ret = add_aio_request(s, aio_req, acb->qiov->iov,
649 acb->qiov->niov, 0, acb->aiocb_type);
650 if (ret < 0) {
651 error_report("add_aio_request is failed");
652 free_aio_req(s, aio_req);
653 if (!acb->nr_pending) {
654 sd_finish_aiocb(acb);
661 * Receive responses of the I/O requests.
663 * This function is registered as a fd handler, and called from the
664 * main loop when s->fd is ready for reading responses.
666 static void coroutine_fn aio_read_response(void *opaque)
668 SheepdogObjRsp rsp;
669 BDRVSheepdogState *s = opaque;
670 int fd = s->fd;
671 int ret;
672 AIOReq *aio_req = NULL;
673 SheepdogAIOCB *acb;
674 unsigned long idx;
676 if (QLIST_EMPTY(&s->inflight_aio_head)) {
677 goto out;
680 /* read a header */
681 ret = qemu_co_recv(fd, &rsp, sizeof(rsp));
682 if (ret < 0) {
683 error_report("failed to get the header, %s", strerror(errno));
684 goto out;
687 /* find the right aio_req from the inflight aio list */
688 QLIST_FOREACH(aio_req, &s->inflight_aio_head, aio_siblings) {
689 if (aio_req->id == rsp.id) {
690 break;
693 if (!aio_req) {
694 error_report("cannot find aio_req %x", rsp.id);
695 goto out;
698 acb = aio_req->aiocb;
700 switch (acb->aiocb_type) {
701 case AIOCB_WRITE_UDATA:
702 /* this coroutine context is no longer suitable for co_recv
703 * because we may send data to update vdi objects */
704 s->co_recv = NULL;
705 if (!is_data_obj(aio_req->oid)) {
706 break;
708 idx = data_oid_to_idx(aio_req->oid);
710 if (s->inode.data_vdi_id[idx] != s->inode.vdi_id) {
712 * If the object is newly created one, we need to update
713 * the vdi object (metadata object). min_dirty_data_idx
714 * and max_dirty_data_idx are changed to include updated
715 * index between them.
717 s->inode.data_vdi_id[idx] = s->inode.vdi_id;
718 s->max_dirty_data_idx = MAX(idx, s->max_dirty_data_idx);
719 s->min_dirty_data_idx = MIN(idx, s->min_dirty_data_idx);
722 * Some requests may be blocked because simultaneous
723 * create requests are not allowed, so we search the
724 * pending requests here.
726 send_pending_req(s, vid_to_data_oid(s->inode.vdi_id, idx));
728 break;
729 case AIOCB_READ_UDATA:
730 ret = qemu_co_recvv(fd, acb->qiov->iov, acb->qiov->niov,
731 aio_req->iov_offset, rsp.data_length);
732 if (ret < 0) {
733 error_report("failed to get the data, %s", strerror(errno));
734 goto out;
736 break;
739 if (rsp.result != SD_RES_SUCCESS) {
740 acb->ret = -EIO;
741 error_report("%s", sd_strerror(rsp.result));
744 free_aio_req(s, aio_req);
745 if (!acb->nr_pending) {
747 * We've finished all requests which belong to the AIOCB, so
748 * we can switch back to sd_co_readv/writev now.
750 acb->aio_done_func(acb);
752 out:
753 s->co_recv = NULL;
756 static void co_read_response(void *opaque)
758 BDRVSheepdogState *s = opaque;
760 if (!s->co_recv) {
761 s->co_recv = qemu_coroutine_create(aio_read_response);
764 qemu_coroutine_enter(s->co_recv, opaque);
767 static void co_write_request(void *opaque)
769 BDRVSheepdogState *s = opaque;
771 qemu_coroutine_enter(s->co_send, NULL);
774 static int aio_flush_request(void *opaque)
776 BDRVSheepdogState *s = opaque;
778 return !QLIST_EMPTY(&s->inflight_aio_head) ||
779 !QLIST_EMPTY(&s->pending_aio_head);
782 static int set_nodelay(int fd)
784 int ret, opt;
786 opt = 1;
787 ret = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, (char *)&opt, sizeof(opt));
788 return ret;
792 * Return a socket discriptor to read/write objects.
794 * We cannot use this discriptor for other operations because
795 * the block driver may be on waiting response from the server.
797 static int get_sheep_fd(BDRVSheepdogState *s)
799 int ret, fd;
801 fd = connect_to_sdog(s->addr, s->port);
802 if (fd < 0) {
803 error_report("%s", strerror(errno));
804 return fd;
807 socket_set_nonblock(fd);
809 ret = set_nodelay(fd);
810 if (ret) {
811 error_report("%s", strerror(errno));
812 closesocket(fd);
813 return -errno;
816 qemu_aio_set_fd_handler(fd, co_read_response, NULL, aio_flush_request, s);
817 return fd;
821 * Parse a filename
823 * filename must be one of the following formats:
824 * 1. [vdiname]
825 * 2. [vdiname]:[snapid]
826 * 3. [vdiname]:[tag]
827 * 4. [hostname]:[port]:[vdiname]
828 * 5. [hostname]:[port]:[vdiname]:[snapid]
829 * 6. [hostname]:[port]:[vdiname]:[tag]
831 * You can boot from the snapshot images by specifying `snapid` or
832 * `tag'.
834 * You can run VMs outside the Sheepdog cluster by specifying
835 * `hostname' and `port' (experimental).
837 static int parse_vdiname(BDRVSheepdogState *s, const char *filename,
838 char *vdi, uint32_t *snapid, char *tag)
840 char *p, *q;
841 int nr_sep;
843 p = q = g_strdup(filename);
845 /* count the number of separators */
846 nr_sep = 0;
847 while (*p) {
848 if (*p == ':') {
849 nr_sep++;
851 p++;
853 p = q;
855 /* use the first two tokens as hostname and port number. */
856 if (nr_sep >= 2) {
857 s->addr = p;
858 p = strchr(p, ':');
859 *p++ = '\0';
861 s->port = p;
862 p = strchr(p, ':');
863 *p++ = '\0';
864 } else {
865 s->addr = NULL;
866 s->port = 0;
869 strncpy(vdi, p, SD_MAX_VDI_LEN);
871 p = strchr(vdi, ':');
872 if (p) {
873 *p++ = '\0';
874 *snapid = strtoul(p, NULL, 10);
875 if (*snapid == 0) {
876 strncpy(tag, p, SD_MAX_VDI_TAG_LEN);
878 } else {
879 *snapid = CURRENT_VDI_ID; /* search current vdi */
882 if (s->addr == NULL) {
883 g_free(q);
886 return 0;
889 static int find_vdi_name(BDRVSheepdogState *s, char *filename, uint32_t snapid,
890 char *tag, uint32_t *vid, int for_snapshot)
892 int ret, fd;
893 SheepdogVdiReq hdr;
894 SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
895 unsigned int wlen, rlen = 0;
896 char buf[SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN];
898 fd = connect_to_sdog(s->addr, s->port);
899 if (fd < 0) {
900 return fd;
903 memset(buf, 0, sizeof(buf));
904 strncpy(buf, filename, SD_MAX_VDI_LEN);
905 strncpy(buf + SD_MAX_VDI_LEN, tag, SD_MAX_VDI_TAG_LEN);
907 memset(&hdr, 0, sizeof(hdr));
908 if (for_snapshot) {
909 hdr.opcode = SD_OP_GET_VDI_INFO;
910 } else {
911 hdr.opcode = SD_OP_LOCK_VDI;
913 wlen = SD_MAX_VDI_LEN + SD_MAX_VDI_TAG_LEN;
914 hdr.proto_ver = SD_PROTO_VER;
915 hdr.data_length = wlen;
916 hdr.snapid = snapid;
917 hdr.flags = SD_FLAG_CMD_WRITE;
919 ret = do_req(fd, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
920 if (ret) {
921 goto out;
924 if (rsp->result != SD_RES_SUCCESS) {
925 error_report("cannot get vdi info, %s, %s %d %s",
926 sd_strerror(rsp->result), filename, snapid, tag);
927 if (rsp->result == SD_RES_NO_VDI) {
928 ret = -ENOENT;
929 } else {
930 ret = -EIO;
932 goto out;
934 *vid = rsp->vdi_id;
936 ret = 0;
937 out:
938 closesocket(fd);
939 return ret;
942 static int coroutine_fn add_aio_request(BDRVSheepdogState *s, AIOReq *aio_req,
943 struct iovec *iov, int niov, int create,
944 enum AIOCBState aiocb_type)
946 int nr_copies = s->inode.nr_copies;
947 SheepdogObjReq hdr;
948 unsigned int wlen;
949 int ret;
950 uint64_t oid = aio_req->oid;
951 unsigned int datalen = aio_req->data_len;
952 uint64_t offset = aio_req->offset;
953 uint8_t flags = aio_req->flags;
954 uint64_t old_oid = aio_req->base_oid;
956 if (!nr_copies) {
957 error_report("bug");
960 memset(&hdr, 0, sizeof(hdr));
962 if (aiocb_type == AIOCB_READ_UDATA) {
963 wlen = 0;
964 hdr.opcode = SD_OP_READ_OBJ;
965 hdr.flags = flags;
966 } else if (create) {
967 wlen = datalen;
968 hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
969 hdr.flags = SD_FLAG_CMD_WRITE | flags;
970 } else {
971 wlen = datalen;
972 hdr.opcode = SD_OP_WRITE_OBJ;
973 hdr.flags = SD_FLAG_CMD_WRITE | flags;
976 if (s->cache_enabled) {
977 hdr.flags |= SD_FLAG_CMD_CACHE;
980 hdr.oid = oid;
981 hdr.cow_oid = old_oid;
982 hdr.copies = s->inode.nr_copies;
984 hdr.data_length = datalen;
985 hdr.offset = offset;
987 hdr.id = aio_req->id;
989 qemu_co_mutex_lock(&s->lock);
990 s->co_send = qemu_coroutine_self();
991 qemu_aio_set_fd_handler(s->fd, co_read_response, co_write_request,
992 aio_flush_request, s);
993 socket_set_cork(s->fd, 1);
995 /* send a header */
996 ret = qemu_co_send(s->fd, &hdr, sizeof(hdr));
997 if (ret < 0) {
998 qemu_co_mutex_unlock(&s->lock);
999 error_report("failed to send a req, %s", strerror(errno));
1000 return -errno;
1003 if (wlen) {
1004 ret = qemu_co_sendv(s->fd, iov, niov, aio_req->iov_offset, wlen);
1005 if (ret < 0) {
1006 qemu_co_mutex_unlock(&s->lock);
1007 error_report("failed to send a data, %s", strerror(errno));
1008 return -errno;
1012 socket_set_cork(s->fd, 0);
1013 qemu_aio_set_fd_handler(s->fd, co_read_response, NULL,
1014 aio_flush_request, s);
1015 qemu_co_mutex_unlock(&s->lock);
1017 return 0;
1020 static int read_write_object(int fd, char *buf, uint64_t oid, int copies,
1021 unsigned int datalen, uint64_t offset,
1022 int write, int create, uint8_t cache)
1024 SheepdogObjReq hdr;
1025 SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
1026 unsigned int wlen, rlen;
1027 int ret;
1029 memset(&hdr, 0, sizeof(hdr));
1031 if (write) {
1032 wlen = datalen;
1033 rlen = 0;
1034 hdr.flags = SD_FLAG_CMD_WRITE;
1035 if (create) {
1036 hdr.opcode = SD_OP_CREATE_AND_WRITE_OBJ;
1037 } else {
1038 hdr.opcode = SD_OP_WRITE_OBJ;
1040 } else {
1041 wlen = 0;
1042 rlen = datalen;
1043 hdr.opcode = SD_OP_READ_OBJ;
1046 if (cache) {
1047 hdr.flags |= SD_FLAG_CMD_CACHE;
1050 hdr.oid = oid;
1051 hdr.data_length = datalen;
1052 hdr.offset = offset;
1053 hdr.copies = copies;
1055 ret = do_req(fd, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
1056 if (ret) {
1057 error_report("failed to send a request to the sheep");
1058 return ret;
1061 switch (rsp->result) {
1062 case SD_RES_SUCCESS:
1063 return 0;
1064 default:
1065 error_report("%s", sd_strerror(rsp->result));
1066 return -EIO;
1070 static int read_object(int fd, char *buf, uint64_t oid, int copies,
1071 unsigned int datalen, uint64_t offset, uint8_t cache)
1073 return read_write_object(fd, buf, oid, copies, datalen, offset, 0, 0,
1074 cache);
1077 static int write_object(int fd, char *buf, uint64_t oid, int copies,
1078 unsigned int datalen, uint64_t offset, int create,
1079 uint8_t cache)
1081 return read_write_object(fd, buf, oid, copies, datalen, offset, 1, create,
1082 cache);
1085 static int sd_open(BlockDriverState *bs, const char *filename, int flags)
1087 int ret, fd;
1088 uint32_t vid = 0;
1089 BDRVSheepdogState *s = bs->opaque;
1090 char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
1091 uint32_t snapid;
1092 char *buf = NULL;
1094 strstart(filename, "sheepdog:", (const char **)&filename);
1096 QLIST_INIT(&s->inflight_aio_head);
1097 QLIST_INIT(&s->pending_aio_head);
1098 s->fd = -1;
1100 memset(vdi, 0, sizeof(vdi));
1101 memset(tag, 0, sizeof(tag));
1102 if (parse_vdiname(s, filename, vdi, &snapid, tag) < 0) {
1103 ret = -EINVAL;
1104 goto out;
1106 s->fd = get_sheep_fd(s);
1107 if (s->fd < 0) {
1108 ret = s->fd;
1109 goto out;
1112 ret = find_vdi_name(s, vdi, snapid, tag, &vid, 0);
1113 if (ret) {
1114 goto out;
1117 if (flags & BDRV_O_CACHE_WB) {
1118 s->cache_enabled = 1;
1119 s->flush_fd = connect_to_sdog(s->addr, s->port);
1120 if (s->flush_fd < 0) {
1121 error_report("failed to connect");
1122 ret = s->flush_fd;
1123 goto out;
1127 if (snapid || tag[0] != '\0') {
1128 dprintf("%" PRIx32 " snapshot inode was open.\n", vid);
1129 s->is_snapshot = 1;
1132 fd = connect_to_sdog(s->addr, s->port);
1133 if (fd < 0) {
1134 error_report("failed to connect");
1135 ret = fd;
1136 goto out;
1139 buf = g_malloc(SD_INODE_SIZE);
1140 ret = read_object(fd, buf, vid_to_vdi_oid(vid), 0, SD_INODE_SIZE, 0,
1141 s->cache_enabled);
1143 closesocket(fd);
1145 if (ret) {
1146 goto out;
1149 memcpy(&s->inode, buf, sizeof(s->inode));
1150 s->min_dirty_data_idx = UINT32_MAX;
1151 s->max_dirty_data_idx = 0;
1153 bs->total_sectors = s->inode.vdi_size / SECTOR_SIZE;
1154 strncpy(s->name, vdi, sizeof(s->name));
1155 qemu_co_mutex_init(&s->lock);
1156 g_free(buf);
1157 return 0;
1158 out:
1159 qemu_aio_set_fd_handler(s->fd, NULL, NULL, NULL, NULL);
1160 if (s->fd >= 0) {
1161 closesocket(s->fd);
1163 g_free(buf);
1164 return ret;
1167 static int do_sd_create(char *filename, int64_t vdi_size,
1168 uint32_t base_vid, uint32_t *vdi_id, int snapshot,
1169 const char *addr, const char *port)
1171 SheepdogVdiReq hdr;
1172 SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
1173 int fd, ret;
1174 unsigned int wlen, rlen = 0;
1175 char buf[SD_MAX_VDI_LEN];
1177 fd = connect_to_sdog(addr, port);
1178 if (fd < 0) {
1179 return fd;
1182 memset(buf, 0, sizeof(buf));
1183 strncpy(buf, filename, SD_MAX_VDI_LEN);
1185 memset(&hdr, 0, sizeof(hdr));
1186 hdr.opcode = SD_OP_NEW_VDI;
1187 hdr.base_vdi_id = base_vid;
1189 wlen = SD_MAX_VDI_LEN;
1191 hdr.flags = SD_FLAG_CMD_WRITE;
1192 hdr.snapid = snapshot;
1194 hdr.data_length = wlen;
1195 hdr.vdi_size = vdi_size;
1197 ret = do_req(fd, (SheepdogReq *)&hdr, buf, &wlen, &rlen);
1199 closesocket(fd);
1201 if (ret) {
1202 return ret;
1205 if (rsp->result != SD_RES_SUCCESS) {
1206 error_report("%s, %s", sd_strerror(rsp->result), filename);
1207 return -EIO;
1210 if (vdi_id) {
1211 *vdi_id = rsp->vdi_id;
1214 return 0;
1217 static int sd_prealloc(const char *filename)
1219 BlockDriverState *bs = NULL;
1220 uint32_t idx, max_idx;
1221 int64_t vdi_size;
1222 void *buf = g_malloc0(SD_DATA_OBJ_SIZE);
1223 int ret;
1225 ret = bdrv_file_open(&bs, filename, BDRV_O_RDWR);
1226 if (ret < 0) {
1227 goto out;
1230 vdi_size = bdrv_getlength(bs);
1231 if (vdi_size < 0) {
1232 ret = vdi_size;
1233 goto out;
1235 max_idx = DIV_ROUND_UP(vdi_size, SD_DATA_OBJ_SIZE);
1237 for (idx = 0; idx < max_idx; idx++) {
1239 * The created image can be a cloned image, so we need to read
1240 * a data from the source image.
1242 ret = bdrv_pread(bs, idx * SD_DATA_OBJ_SIZE, buf, SD_DATA_OBJ_SIZE);
1243 if (ret < 0) {
1244 goto out;
1246 ret = bdrv_pwrite(bs, idx * SD_DATA_OBJ_SIZE, buf, SD_DATA_OBJ_SIZE);
1247 if (ret < 0) {
1248 goto out;
1251 out:
1252 if (bs) {
1253 bdrv_delete(bs);
1255 g_free(buf);
1257 return ret;
1260 static int sd_create(const char *filename, QEMUOptionParameter *options)
1262 int ret = 0;
1263 uint32_t vid = 0, base_vid = 0;
1264 int64_t vdi_size = 0;
1265 char *backing_file = NULL;
1266 BDRVSheepdogState *s;
1267 char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
1268 uint32_t snapid;
1269 int prealloc = 0;
1270 const char *vdiname;
1272 s = g_malloc0(sizeof(BDRVSheepdogState));
1274 strstart(filename, "sheepdog:", &vdiname);
1276 memset(vdi, 0, sizeof(vdi));
1277 memset(tag, 0, sizeof(tag));
1278 if (parse_vdiname(s, vdiname, vdi, &snapid, tag) < 0) {
1279 error_report("invalid filename");
1280 ret = -EINVAL;
1281 goto out;
1284 while (options && options->name) {
1285 if (!strcmp(options->name, BLOCK_OPT_SIZE)) {
1286 vdi_size = options->value.n;
1287 } else if (!strcmp(options->name, BLOCK_OPT_BACKING_FILE)) {
1288 backing_file = options->value.s;
1289 } else if (!strcmp(options->name, BLOCK_OPT_PREALLOC)) {
1290 if (!options->value.s || !strcmp(options->value.s, "off")) {
1291 prealloc = 0;
1292 } else if (!strcmp(options->value.s, "full")) {
1293 prealloc = 1;
1294 } else {
1295 error_report("Invalid preallocation mode: '%s'",
1296 options->value.s);
1297 ret = -EINVAL;
1298 goto out;
1301 options++;
1304 if (vdi_size > SD_MAX_VDI_SIZE) {
1305 error_report("too big image size");
1306 ret = -EINVAL;
1307 goto out;
1310 if (backing_file) {
1311 BlockDriverState *bs;
1312 BDRVSheepdogState *s;
1313 BlockDriver *drv;
1315 /* Currently, only Sheepdog backing image is supported. */
1316 drv = bdrv_find_protocol(backing_file);
1317 if (!drv || strcmp(drv->protocol_name, "sheepdog") != 0) {
1318 error_report("backing_file must be a sheepdog image");
1319 ret = -EINVAL;
1320 goto out;
1323 ret = bdrv_file_open(&bs, backing_file, 0);
1324 if (ret < 0) {
1325 goto out;
1328 s = bs->opaque;
1330 if (!is_snapshot(&s->inode)) {
1331 error_report("cannot clone from a non snapshot vdi");
1332 bdrv_delete(bs);
1333 ret = -EINVAL;
1334 goto out;
1337 base_vid = s->inode.vdi_id;
1338 bdrv_delete(bs);
1341 ret = do_sd_create(vdi, vdi_size, base_vid, &vid, 0, s->addr, s->port);
1342 if (!prealloc || ret) {
1343 goto out;
1346 ret = sd_prealloc(filename);
1347 out:
1348 g_free(s);
1349 return ret;
1352 static void sd_close(BlockDriverState *bs)
1354 BDRVSheepdogState *s = bs->opaque;
1355 SheepdogVdiReq hdr;
1356 SheepdogVdiRsp *rsp = (SheepdogVdiRsp *)&hdr;
1357 unsigned int wlen, rlen = 0;
1358 int fd, ret;
1360 dprintf("%s\n", s->name);
1362 fd = connect_to_sdog(s->addr, s->port);
1363 if (fd < 0) {
1364 return;
1367 memset(&hdr, 0, sizeof(hdr));
1369 hdr.opcode = SD_OP_RELEASE_VDI;
1370 wlen = strlen(s->name) + 1;
1371 hdr.data_length = wlen;
1372 hdr.flags = SD_FLAG_CMD_WRITE;
1374 ret = do_req(fd, (SheepdogReq *)&hdr, s->name, &wlen, &rlen);
1376 closesocket(fd);
1378 if (!ret && rsp->result != SD_RES_SUCCESS &&
1379 rsp->result != SD_RES_VDI_NOT_LOCKED) {
1380 error_report("%s, %s", sd_strerror(rsp->result), s->name);
1383 qemu_aio_set_fd_handler(s->fd, NULL, NULL, NULL, NULL);
1384 closesocket(s->fd);
1385 if (s->cache_enabled) {
1386 closesocket(s->flush_fd);
1388 g_free(s->addr);
1391 static int64_t sd_getlength(BlockDriverState *bs)
1393 BDRVSheepdogState *s = bs->opaque;
1395 return s->inode.vdi_size;
1398 static int sd_truncate(BlockDriverState *bs, int64_t offset)
1400 BDRVSheepdogState *s = bs->opaque;
1401 int ret, fd;
1402 unsigned int datalen;
1404 if (offset < s->inode.vdi_size) {
1405 error_report("shrinking is not supported");
1406 return -EINVAL;
1407 } else if (offset > SD_MAX_VDI_SIZE) {
1408 error_report("too big image size");
1409 return -EINVAL;
1412 fd = connect_to_sdog(s->addr, s->port);
1413 if (fd < 0) {
1414 return fd;
1417 /* we don't need to update entire object */
1418 datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
1419 s->inode.vdi_size = offset;
1420 ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
1421 s->inode.nr_copies, datalen, 0, 0, s->cache_enabled);
1422 close(fd);
1424 if (ret < 0) {
1425 error_report("failed to update an inode.");
1428 return ret;
1432 * This function is called after writing data objects. If we need to
1433 * update metadata, this sends a write request to the vdi object.
1434 * Otherwise, this switches back to sd_co_readv/writev.
1436 static void coroutine_fn sd_write_done(SheepdogAIOCB *acb)
1438 int ret;
1439 BDRVSheepdogState *s = acb->common.bs->opaque;
1440 struct iovec iov;
1441 AIOReq *aio_req;
1442 uint32_t offset, data_len, mn, mx;
1444 mn = s->min_dirty_data_idx;
1445 mx = s->max_dirty_data_idx;
1446 if (mn <= mx) {
1447 /* we need to update the vdi object. */
1448 offset = sizeof(s->inode) - sizeof(s->inode.data_vdi_id) +
1449 mn * sizeof(s->inode.data_vdi_id[0]);
1450 data_len = (mx - mn + 1) * sizeof(s->inode.data_vdi_id[0]);
1452 s->min_dirty_data_idx = UINT32_MAX;
1453 s->max_dirty_data_idx = 0;
1455 iov.iov_base = &s->inode;
1456 iov.iov_len = sizeof(s->inode);
1457 aio_req = alloc_aio_req(s, acb, vid_to_vdi_oid(s->inode.vdi_id),
1458 data_len, offset, 0, 0, offset);
1459 QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
1460 ret = add_aio_request(s, aio_req, &iov, 1, 0, AIOCB_WRITE_UDATA);
1461 if (ret) {
1462 free_aio_req(s, aio_req);
1463 acb->ret = -EIO;
1464 goto out;
1467 acb->aio_done_func = sd_finish_aiocb;
1468 acb->aiocb_type = AIOCB_WRITE_UDATA;
1469 return;
1471 out:
1472 sd_finish_aiocb(acb);
1476 * Create a writable VDI from a snapshot
1478 static int sd_create_branch(BDRVSheepdogState *s)
1480 int ret, fd;
1481 uint32_t vid;
1482 char *buf;
1484 dprintf("%" PRIx32 " is snapshot.\n", s->inode.vdi_id);
1486 buf = g_malloc(SD_INODE_SIZE);
1488 ret = do_sd_create(s->name, s->inode.vdi_size, s->inode.vdi_id, &vid, 1,
1489 s->addr, s->port);
1490 if (ret) {
1491 goto out;
1494 dprintf("%" PRIx32 " is created.\n", vid);
1496 fd = connect_to_sdog(s->addr, s->port);
1497 if (fd < 0) {
1498 error_report("failed to connect");
1499 ret = fd;
1500 goto out;
1503 ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
1504 SD_INODE_SIZE, 0, s->cache_enabled);
1506 closesocket(fd);
1508 if (ret < 0) {
1509 goto out;
1512 memcpy(&s->inode, buf, sizeof(s->inode));
1514 s->is_snapshot = 0;
1515 ret = 0;
1516 dprintf("%" PRIx32 " was newly created.\n", s->inode.vdi_id);
1518 out:
1519 g_free(buf);
1521 return ret;
1525 * Send I/O requests to the server.
1527 * This function sends requests to the server, links the requests to
1528 * the inflight_list in BDRVSheepdogState, and exits without
1529 * waiting the response. The responses are received in the
1530 * `aio_read_response' function which is called from the main loop as
1531 * a fd handler.
1533 * Returns 1 when we need to wait a response, 0 when there is no sent
1534 * request and -errno in error cases.
1536 static int coroutine_fn sd_co_rw_vector(void *p)
1538 SheepdogAIOCB *acb = p;
1539 int ret = 0;
1540 unsigned long len, done = 0, total = acb->nb_sectors * SECTOR_SIZE;
1541 unsigned long idx = acb->sector_num * SECTOR_SIZE / SD_DATA_OBJ_SIZE;
1542 uint64_t oid;
1543 uint64_t offset = (acb->sector_num * SECTOR_SIZE) % SD_DATA_OBJ_SIZE;
1544 BDRVSheepdogState *s = acb->common.bs->opaque;
1545 SheepdogInode *inode = &s->inode;
1546 AIOReq *aio_req;
1548 if (acb->aiocb_type == AIOCB_WRITE_UDATA && s->is_snapshot) {
1550 * In the case we open the snapshot VDI, Sheepdog creates the
1551 * writable VDI when we do a write operation first.
1553 ret = sd_create_branch(s);
1554 if (ret) {
1555 acb->ret = -EIO;
1556 goto out;
1561 * Make sure we don't free the aiocb before we are done with all requests.
1562 * This additional reference is dropped at the end of this function.
1564 acb->nr_pending++;
1566 while (done != total) {
1567 uint8_t flags = 0;
1568 uint64_t old_oid = 0;
1569 int create = 0;
1571 oid = vid_to_data_oid(inode->data_vdi_id[idx], idx);
1573 len = MIN(total - done, SD_DATA_OBJ_SIZE - offset);
1575 switch (acb->aiocb_type) {
1576 case AIOCB_READ_UDATA:
1577 if (!inode->data_vdi_id[idx]) {
1578 qemu_iovec_memset(acb->qiov, done, 0, len);
1579 goto done;
1581 break;
1582 case AIOCB_WRITE_UDATA:
1583 if (!inode->data_vdi_id[idx]) {
1584 create = 1;
1585 } else if (!is_data_obj_writable(inode, idx)) {
1586 /* Copy-On-Write */
1587 create = 1;
1588 old_oid = oid;
1589 flags = SD_FLAG_CMD_COW;
1591 break;
1592 default:
1593 break;
1596 if (create) {
1597 dprintf("update ino (%" PRIu32 ") %" PRIu64 " %" PRIu64 " %ld\n",
1598 inode->vdi_id, oid,
1599 vid_to_data_oid(inode->data_vdi_id[idx], idx), idx);
1600 oid = vid_to_data_oid(inode->vdi_id, idx);
1601 dprintf("new oid %" PRIx64 "\n", oid);
1604 aio_req = alloc_aio_req(s, acb, oid, len, offset, flags, old_oid, done);
1606 if (create) {
1607 AIOReq *areq;
1608 QLIST_FOREACH(areq, &s->inflight_aio_head, aio_siblings) {
1609 if (areq->oid == oid) {
1611 * Sheepdog cannot handle simultaneous create
1612 * requests to the same object. So we cannot send
1613 * the request until the previous request
1614 * finishes.
1616 aio_req->flags = 0;
1617 aio_req->base_oid = 0;
1618 QLIST_INSERT_HEAD(&s->pending_aio_head, aio_req,
1619 aio_siblings);
1620 goto done;
1625 QLIST_INSERT_HEAD(&s->inflight_aio_head, aio_req, aio_siblings);
1626 ret = add_aio_request(s, aio_req, acb->qiov->iov, acb->qiov->niov,
1627 create, acb->aiocb_type);
1628 if (ret < 0) {
1629 error_report("add_aio_request is failed");
1630 free_aio_req(s, aio_req);
1631 acb->ret = -EIO;
1632 goto out;
1634 done:
1635 offset = 0;
1636 idx++;
1637 done += len;
1639 out:
1640 if (!--acb->nr_pending) {
1641 return acb->ret;
1643 return 1;
1646 static coroutine_fn int sd_co_writev(BlockDriverState *bs, int64_t sector_num,
1647 int nb_sectors, QEMUIOVector *qiov)
1649 SheepdogAIOCB *acb;
1650 int ret;
1652 if (bs->growable && sector_num + nb_sectors > bs->total_sectors) {
1653 ret = sd_truncate(bs, (sector_num + nb_sectors) * SECTOR_SIZE);
1654 if (ret < 0) {
1655 return ret;
1657 bs->total_sectors = sector_num + nb_sectors;
1660 acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
1661 acb->aio_done_func = sd_write_done;
1662 acb->aiocb_type = AIOCB_WRITE_UDATA;
1664 ret = sd_co_rw_vector(acb);
1665 if (ret <= 0) {
1666 qemu_aio_release(acb);
1667 return ret;
1670 qemu_coroutine_yield();
1672 return acb->ret;
1675 static coroutine_fn int sd_co_readv(BlockDriverState *bs, int64_t sector_num,
1676 int nb_sectors, QEMUIOVector *qiov)
1678 SheepdogAIOCB *acb;
1679 int ret;
1681 acb = sd_aio_setup(bs, qiov, sector_num, nb_sectors, NULL, NULL);
1682 acb->aiocb_type = AIOCB_READ_UDATA;
1683 acb->aio_done_func = sd_finish_aiocb;
1685 ret = sd_co_rw_vector(acb);
1686 if (ret <= 0) {
1687 qemu_aio_release(acb);
1688 return ret;
1691 qemu_coroutine_yield();
1693 return acb->ret;
1696 static int coroutine_fn sd_co_flush_to_disk(BlockDriverState *bs)
1698 BDRVSheepdogState *s = bs->opaque;
1699 SheepdogObjReq hdr = { 0 };
1700 SheepdogObjRsp *rsp = (SheepdogObjRsp *)&hdr;
1701 SheepdogInode *inode = &s->inode;
1702 int ret;
1703 unsigned int wlen = 0, rlen = 0;
1705 if (!s->cache_enabled) {
1706 return 0;
1709 hdr.opcode = SD_OP_FLUSH_VDI;
1710 hdr.oid = vid_to_vdi_oid(inode->vdi_id);
1712 ret = do_req(s->flush_fd, (SheepdogReq *)&hdr, NULL, &wlen, &rlen);
1713 if (ret) {
1714 error_report("failed to send a request to the sheep");
1715 return ret;
1718 if (rsp->result == SD_RES_INVALID_PARMS) {
1719 dprintf("disable write cache since the server doesn't support it\n");
1721 s->cache_enabled = 0;
1722 closesocket(s->flush_fd);
1723 return 0;
1726 if (rsp->result != SD_RES_SUCCESS) {
1727 error_report("%s", sd_strerror(rsp->result));
1728 return -EIO;
1731 return 0;
1734 static int sd_snapshot_create(BlockDriverState *bs, QEMUSnapshotInfo *sn_info)
1736 BDRVSheepdogState *s = bs->opaque;
1737 int ret, fd;
1738 uint32_t new_vid;
1739 SheepdogInode *inode;
1740 unsigned int datalen;
1742 dprintf("sn_info: name %s id_str %s s: name %s vm_state_size %" PRId64 " "
1743 "is_snapshot %d\n", sn_info->name, sn_info->id_str,
1744 s->name, sn_info->vm_state_size, s->is_snapshot);
1746 if (s->is_snapshot) {
1747 error_report("You can't create a snapshot of a snapshot VDI, "
1748 "%s (%" PRIu32 ").", s->name, s->inode.vdi_id);
1750 return -EINVAL;
1753 dprintf("%s %s\n", sn_info->name, sn_info->id_str);
1755 s->inode.vm_state_size = sn_info->vm_state_size;
1756 s->inode.vm_clock_nsec = sn_info->vm_clock_nsec;
1757 strncpy(s->inode.tag, sn_info->name, sizeof(s->inode.tag));
1758 /* we don't need to update entire object */
1759 datalen = SD_INODE_SIZE - sizeof(s->inode.data_vdi_id);
1761 /* refresh inode. */
1762 fd = connect_to_sdog(s->addr, s->port);
1763 if (fd < 0) {
1764 ret = fd;
1765 goto cleanup;
1768 ret = write_object(fd, (char *)&s->inode, vid_to_vdi_oid(s->inode.vdi_id),
1769 s->inode.nr_copies, datalen, 0, 0, s->cache_enabled);
1770 if (ret < 0) {
1771 error_report("failed to write snapshot's inode.");
1772 goto cleanup;
1775 ret = do_sd_create(s->name, s->inode.vdi_size, s->inode.vdi_id, &new_vid, 1,
1776 s->addr, s->port);
1777 if (ret < 0) {
1778 error_report("failed to create inode for snapshot. %s",
1779 strerror(errno));
1780 goto cleanup;
1783 inode = (SheepdogInode *)g_malloc(datalen);
1785 ret = read_object(fd, (char *)inode, vid_to_vdi_oid(new_vid),
1786 s->inode.nr_copies, datalen, 0, s->cache_enabled);
1788 if (ret < 0) {
1789 error_report("failed to read new inode info. %s", strerror(errno));
1790 goto cleanup;
1793 memcpy(&s->inode, inode, datalen);
1794 dprintf("s->inode: name %s snap_id %x oid %x\n",
1795 s->inode.name, s->inode.snap_id, s->inode.vdi_id);
1797 cleanup:
1798 closesocket(fd);
1799 return ret;
1802 static int sd_snapshot_goto(BlockDriverState *bs, const char *snapshot_id)
1804 BDRVSheepdogState *s = bs->opaque;
1805 BDRVSheepdogState *old_s;
1806 char vdi[SD_MAX_VDI_LEN], tag[SD_MAX_VDI_TAG_LEN];
1807 char *buf = NULL;
1808 uint32_t vid;
1809 uint32_t snapid = 0;
1810 int ret = 0, fd;
1812 old_s = g_malloc(sizeof(BDRVSheepdogState));
1814 memcpy(old_s, s, sizeof(BDRVSheepdogState));
1816 memset(vdi, 0, sizeof(vdi));
1817 strncpy(vdi, s->name, sizeof(vdi));
1819 memset(tag, 0, sizeof(tag));
1820 snapid = strtoul(snapshot_id, NULL, 10);
1821 if (!snapid) {
1822 strncpy(tag, s->name, sizeof(tag));
1825 ret = find_vdi_name(s, vdi, snapid, tag, &vid, 1);
1826 if (ret) {
1827 error_report("Failed to find_vdi_name");
1828 goto out;
1831 fd = connect_to_sdog(s->addr, s->port);
1832 if (fd < 0) {
1833 error_report("failed to connect");
1834 ret = fd;
1835 goto out;
1838 buf = g_malloc(SD_INODE_SIZE);
1839 ret = read_object(fd, buf, vid_to_vdi_oid(vid), s->inode.nr_copies,
1840 SD_INODE_SIZE, 0, s->cache_enabled);
1842 closesocket(fd);
1844 if (ret) {
1845 goto out;
1848 memcpy(&s->inode, buf, sizeof(s->inode));
1850 if (!s->inode.vm_state_size) {
1851 error_report("Invalid snapshot");
1852 ret = -ENOENT;
1853 goto out;
1856 s->is_snapshot = 1;
1858 g_free(buf);
1859 g_free(old_s);
1861 return 0;
1862 out:
1863 /* recover bdrv_sd_state */
1864 memcpy(s, old_s, sizeof(BDRVSheepdogState));
1865 g_free(buf);
1866 g_free(old_s);
1868 error_report("failed to open. recover old bdrv_sd_state.");
1870 return ret;
1873 static int sd_snapshot_delete(BlockDriverState *bs, const char *snapshot_id)
1875 /* FIXME: Delete specified snapshot id. */
1876 return 0;
1879 static int sd_snapshot_list(BlockDriverState *bs, QEMUSnapshotInfo **psn_tab)
1881 BDRVSheepdogState *s = bs->opaque;
1882 SheepdogReq req;
1883 int fd, nr = 1024, ret, max = BITS_TO_LONGS(SD_NR_VDIS) * sizeof(long);
1884 QEMUSnapshotInfo *sn_tab = NULL;
1885 unsigned wlen, rlen;
1886 int found = 0;
1887 static SheepdogInode inode;
1888 unsigned long *vdi_inuse;
1889 unsigned int start_nr;
1890 uint64_t hval;
1891 uint32_t vid;
1893 vdi_inuse = g_malloc(max);
1895 fd = connect_to_sdog(s->addr, s->port);
1896 if (fd < 0) {
1897 ret = fd;
1898 goto out;
1901 rlen = max;
1902 wlen = 0;
1904 memset(&req, 0, sizeof(req));
1906 req.opcode = SD_OP_READ_VDIS;
1907 req.data_length = max;
1909 ret = do_req(fd, (SheepdogReq *)&req, vdi_inuse, &wlen, &rlen);
1911 closesocket(fd);
1912 if (ret) {
1913 goto out;
1916 sn_tab = g_malloc0(nr * sizeof(*sn_tab));
1918 /* calculate a vdi id with hash function */
1919 hval = fnv_64a_buf(s->name, strlen(s->name), FNV1A_64_INIT);
1920 start_nr = hval & (SD_NR_VDIS - 1);
1922 fd = connect_to_sdog(s->addr, s->port);
1923 if (fd < 0) {
1924 error_report("failed to connect");
1925 ret = fd;
1926 goto out;
1929 for (vid = start_nr; found < nr; vid = (vid + 1) % SD_NR_VDIS) {
1930 if (!test_bit(vid, vdi_inuse)) {
1931 break;
1934 /* we don't need to read entire object */
1935 ret = read_object(fd, (char *)&inode, vid_to_vdi_oid(vid),
1936 0, SD_INODE_SIZE - sizeof(inode.data_vdi_id), 0,
1937 s->cache_enabled);
1939 if (ret) {
1940 continue;
1943 if (!strcmp(inode.name, s->name) && is_snapshot(&inode)) {
1944 sn_tab[found].date_sec = inode.snap_ctime >> 32;
1945 sn_tab[found].date_nsec = inode.snap_ctime & 0xffffffff;
1946 sn_tab[found].vm_state_size = inode.vm_state_size;
1947 sn_tab[found].vm_clock_nsec = inode.vm_clock_nsec;
1949 snprintf(sn_tab[found].id_str, sizeof(sn_tab[found].id_str), "%u",
1950 inode.snap_id);
1951 strncpy(sn_tab[found].name, inode.tag,
1952 MIN(sizeof(sn_tab[found].name), sizeof(inode.tag)));
1953 found++;
1957 closesocket(fd);
1958 out:
1959 *psn_tab = sn_tab;
1961 g_free(vdi_inuse);
1963 if (ret < 0) {
1964 return ret;
1967 return found;
1970 static int do_load_save_vmstate(BDRVSheepdogState *s, uint8_t *data,
1971 int64_t pos, int size, int load)
1973 int fd, create;
1974 int ret = 0, remaining = size;
1975 unsigned int data_len;
1976 uint64_t vmstate_oid;
1977 uint32_t vdi_index;
1978 uint64_t offset;
1980 fd = connect_to_sdog(s->addr, s->port);
1981 if (fd < 0) {
1982 return fd;
1985 while (remaining) {
1986 vdi_index = pos / SD_DATA_OBJ_SIZE;
1987 offset = pos % SD_DATA_OBJ_SIZE;
1989 data_len = MIN(remaining, SD_DATA_OBJ_SIZE);
1991 vmstate_oid = vid_to_vmstate_oid(s->inode.vdi_id, vdi_index);
1993 create = (offset == 0);
1994 if (load) {
1995 ret = read_object(fd, (char *)data, vmstate_oid,
1996 s->inode.nr_copies, data_len, offset,
1997 s->cache_enabled);
1998 } else {
1999 ret = write_object(fd, (char *)data, vmstate_oid,
2000 s->inode.nr_copies, data_len, offset, create,
2001 s->cache_enabled);
2004 if (ret < 0) {
2005 error_report("failed to save vmstate %s", strerror(errno));
2006 goto cleanup;
2009 pos += data_len;
2010 remaining -= data_len;
2012 ret = size;
2013 cleanup:
2014 closesocket(fd);
2015 return ret;
2018 static int sd_save_vmstate(BlockDriverState *bs, const uint8_t *data,
2019 int64_t pos, int size)
2021 BDRVSheepdogState *s = bs->opaque;
2023 return do_load_save_vmstate(s, (uint8_t *)data, pos, size, 0);
2026 static int sd_load_vmstate(BlockDriverState *bs, uint8_t *data,
2027 int64_t pos, int size)
2029 BDRVSheepdogState *s = bs->opaque;
2031 return do_load_save_vmstate(s, data, pos, size, 1);
2035 static QEMUOptionParameter sd_create_options[] = {
2037 .name = BLOCK_OPT_SIZE,
2038 .type = OPT_SIZE,
2039 .help = "Virtual disk size"
2042 .name = BLOCK_OPT_BACKING_FILE,
2043 .type = OPT_STRING,
2044 .help = "File name of a base image"
2047 .name = BLOCK_OPT_PREALLOC,
2048 .type = OPT_STRING,
2049 .help = "Preallocation mode (allowed values: off, full)"
2051 { NULL }
2054 BlockDriver bdrv_sheepdog = {
2055 .format_name = "sheepdog",
2056 .protocol_name = "sheepdog",
2057 .instance_size = sizeof(BDRVSheepdogState),
2058 .bdrv_file_open = sd_open,
2059 .bdrv_close = sd_close,
2060 .bdrv_create = sd_create,
2061 .bdrv_getlength = sd_getlength,
2062 .bdrv_truncate = sd_truncate,
2064 .bdrv_co_readv = sd_co_readv,
2065 .bdrv_co_writev = sd_co_writev,
2066 .bdrv_co_flush_to_disk = sd_co_flush_to_disk,
2068 .bdrv_snapshot_create = sd_snapshot_create,
2069 .bdrv_snapshot_goto = sd_snapshot_goto,
2070 .bdrv_snapshot_delete = sd_snapshot_delete,
2071 .bdrv_snapshot_list = sd_snapshot_list,
2073 .bdrv_save_vmstate = sd_save_vmstate,
2074 .bdrv_load_vmstate = sd_load_vmstate,
2076 .create_options = sd_create_options,
2079 static void bdrv_sheepdog_init(void)
2081 bdrv_register(&bdrv_sheepdog);
2083 block_init(bdrv_sheepdog_init);