?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kernel - Improve physio performance
[dragonfly.git] / sys / kern / subr_diskiocom.c
blobf4e2bfbb77767d9967c451fb4748e81cfcca172f
1 /*
2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/proc.h>
38 #include <sys/sysctl.h>
39 #include <sys/buf.h>
40 #include <sys/conf.h>
41 #include <sys/disklabel.h>
42 #include <sys/disklabel32.h>
43 #include <sys/disklabel64.h>
44 #include <sys/diskslice.h>
45 #include <sys/diskmbr.h>
46 #include <sys/disk.h>
47 #include <sys/malloc.h>
48 #include <sys/device.h>
49 #include <sys/devfs.h>
50 #include <sys/thread.h>
51 #include <sys/queue.h>
52 #include <sys/lock.h>
53 #include <sys/stat.h>
54 #include <sys/uuid.h>
55 #include <sys/dmsg.h>
56
57 #include <sys/buf2.h>
58 #include <sys/mplock2.h>
59 #include <sys/msgport2.h>
60 #include <sys/thread2.h>
61
62 struct dios_open {
63 int openrd;
64 int openwr;
65 };
66
67 struct dios_io {
68 int count;
69 int eof;
70 kdmsg_data_t data;
71 };
72
73 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg");
74
75 static int blk_active;
76 SYSCTL_INT(_debug, OID_AUTO, blk_active, CTLFLAG_RW, &blk_active, 0,
77 "Number of active iocom IOs");
78
79 static int disk_iocom_reconnect(struct disk *dp, struct file *fp);
80 static int disk_rcvdmsg(kdmsg_msg_t *msg);
81
82 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg);
83 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg);
84 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg);
85 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg);
86 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg);
87 static void diskiodone(struct bio *bio);
88
89 void
90 disk_iocom_init(struct disk *dp)
91 {
92 kdmsg_iocom_init(&dp->d_iocom, dp,
93 KDMSG_IOCOMF_AUTOCONN |
94 KDMSG_IOCOMF_AUTORXSPAN |
95 KDMSG_IOCOMF_AUTOTXSPAN,
96 M_DMSG_DISK, disk_rcvdmsg);
97 }
98
99 void
100 disk_iocom_update(struct disk *dp)
101 {
102 }
103
104 void
105 disk_iocom_uninit(struct disk *dp)
106 {
107 kdmsg_iocom_uninit(&dp->d_iocom);
108 }
109
110 int
111 disk_iocom_ioctl(struct disk *dp, int cmd, void *data)
112 {
113 struct file *fp;
114 struct disk_ioc_recluster *recl;
115 int error;
116
117 switch(cmd) {
118 case DIOCRECLUSTER:
119 recl = data;
120 fp = holdfp(curproc->p_fd, recl->fd, -1);
121 if (fp) {
122 error = disk_iocom_reconnect(dp, fp);
123 } else {
124 error = EINVAL;
125 }
126 break;
127 default:
128 error = EOPNOTSUPP;
129 break;
130 }
131 return error;
132 }
133
134 static
135 int
136 disk_iocom_reconnect(struct disk *dp, struct file *fp)
137 {
138 char devname[64];
139
140 ksnprintf(devname, sizeof(devname), "%s%d",
141 dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev));
142
143 kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname);
144
145 dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1;
146 dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK;
147 dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK;
148 dp->d_iocom.auto_lnk_conn.peer_mask = (uint64_t)-1;
149 #if 0
150 if (dp->d_info.d_serialno) {
151 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
152 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
153 "%s/%s", hostname, dp->d_info.d_serialno);
154 } else {
155 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
156 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
157 "%s/%s", hostname, devname);
158 }
159 #endif
160 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
161 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
162 "%s/%s", hostname, devname);
163
164 dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1;
165 dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK;
166 dp->d_iocom.auto_lnk_span.media.block.bytes =
167 dp->d_info.d_media_size;
168 dp->d_iocom.auto_lnk_span.media.block.blksize =
169 dp->d_info.d_media_blksize;
170 ksnprintf(dp->d_iocom.auto_lnk_span.peer_label,
171 sizeof(dp->d_iocom.auto_lnk_span.peer_label),
172 "%s", dp->d_iocom.auto_lnk_conn.peer_label);
173 if (dp->d_info.d_serialno) {
174 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
175 sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
176 "%s", dp->d_info.d_serialno);
177 } else {
178 /*
179 * If no serial number is available generate a dummy serial
180 * number from the host and device name and pray. This will
181 * allow e.g. /dev/vn* to look meaningful on a remote machine.
182 */
183 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
184 sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
185 "%s.%s", hostname, devname);
186 }
187
188 kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL);
189
190 return (0);
191 }
192
193 static int
194 disk_rcvdmsg(kdmsg_msg_t *msg)
195 {
196 struct disk *dp = msg->state->iocom->handle;
197
198 /*
199 * Handle debug messages (these might not be in transactions)
200 */
201 switch(msg->any.head.cmd & DMSGF_CMDSWMASK) {
202 case DMSG_DBG_SHELL:
203 /*
204 * Execute shell command (not supported atm)
205 */
206 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
207 return(0);
208 case DMSG_DBG_SHELL | DMSGF_REPLY:
209 if (msg->aux_data) {
210 msg->aux_data[msg->aux_size - 1] = 0;
211 kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data);
212 }
213 return(0);
214 }
215
216 /*
217 * All remaining messages must be in a transaction.
218 *
219 * NOTE! We currently don't care if the transaction is just
220 * the span transaction (for disk probes) or if it is the
221 * BLK_OPEN transaction.
222 *
223 * NOTE! We are switching on the first message's command. The
224 * actual message command within the transaction may be
225 * different (if streaming within a transaction).
226 */
227 if (msg->state == &msg->state->iocom->state0) {
228 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
229 return(0);
230 }
231
232 switch(msg->state->rxcmd & DMSGF_CMDSWMASK) {
233 case DMSG_BLK_OPEN:
234 disk_blk_open(dp, msg);
235 break;
236 case DMSG_BLK_READ:
237 /*
238 * not reached normally but leave in for completeness
239 */
240 disk_blk_read(dp, msg);
241 break;
242 case DMSG_BLK_WRITE:
243 disk_blk_write(dp, msg);
244 break;
245 case DMSG_BLK_FLUSH:
246 disk_blk_flush(dp, msg);
247 break;
248 case DMSG_BLK_FREEBLKS:
249 disk_blk_freeblks(dp, msg);
250 break;
251 default:
252 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) {
253 if (msg->any.head.cmd & DMSGF_DELETE)
254 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
255 else
256 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP);
257 }
258 break;
259 }
260 return (0);
261 }
262
263 static
264 void
265 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg)
266 {
267 struct dios_open *openst;
268 int error = DMSG_ERR_NOSUPP;
269 int fflags;
270
271 openst = msg->state->any.any;
272 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) {
273 if (openst == NULL) {
274 openst = kmalloc(sizeof(*openst), M_DEVBUF,
275 M_WAITOK | M_ZERO);
276 msg->state->any.any = openst;
277 }
278 fflags = 0;
279 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
280 fflags = FREAD;
281 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
282 fflags |= FWRITE;
283 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR, proc0.p_ucred, NULL);
284 if (error) {
285 error = DMSG_ERR_IO;
286 } else {
287 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
288 ++openst->openrd;
289 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
290 ++openst->openwr;
291 }
292 }
293 #if 0
294 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE &&
295 openst) {
296 fflags = 0;
297 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) &&
298 openst->openrd) {
299 fflags = FREAD;
300 }
301 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) &&
302 openst->openwr) {
303 fflags |= FWRITE;
304 }
305 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL);
306 if (error) {
307 error = DMSG_ERR_IO;
308 } else {
309 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
310 --openst->openrd;
311 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
312 --openst->openwr;
313 }
314 }
315 #endif
316 if (msg->any.head.cmd & DMSGF_DELETE) {
317 if (openst) {
318 while (openst->openrd && openst->openwr) {
319 --openst->openrd;
320 --openst->openwr;
321 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL);
322 }
323 while (openst->openrd) {
324 --openst->openrd;
325 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL);
326 }
327 while (openst->openwr) {
328 --openst->openwr;
329 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL);
330 }
331 kfree(openst, M_DEVBUF);
332 msg->state->any.any = NULL;
333 }
334 kdmsg_msg_reply(msg, error);
335 } else {
336 kdmsg_msg_result(msg, error);
337 }
338 }
339
340 static
341 void
342 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg)
343 {
344 struct dios_io *iost;
345 struct buf *bp;
346 struct bio *bio;
347 int error = DMSG_ERR_NOSUPP;
348 int reterr = 1;
349
350 /*
351 * Only DMSG_BLK_READ commands imply read ops.
352 */
353 iost = msg->state->any.any;
354 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) {
355 if (msg->any.blk_read.bytes < DEV_BSIZE ||
356 msg->any.blk_read.bytes > MAXPHYS) {
357 error = DMSG_ERR_PARAM;
358 goto done;
359 }
360 if (iost == NULL) {
361 iost = kmalloc(sizeof(*iost), M_DEVBUF,
362 M_WAITOK | M_ZERO);
363 msg->state->any.any = iost;
364 }
365 reterr = 0;
366 bp = geteblk(msg->any.blk_read.bytes);
367 bio = &bp->b_bio1;
368 bp->b_cmd = BUF_CMD_READ;
369 bp->b_bcount = msg->any.blk_read.bytes;
370 bp->b_resid = bp->b_bcount;
371 bio->bio_offset = msg->any.blk_read.offset;
372 bio->bio_caller_info1.ptr = msg->state;
373 bio->bio_done = diskiodone;
374
375 /* kdmsg_state_hold(msg->state); */
376 atomic_add_int(&blk_active, 1);
377 atomic_add_int(&iost->count, 1);
378 if (msg->any.head.cmd & DMSGF_DELETE)
379 iost->eof = 1;
380 BUF_KERNPROC(bp);
381 dev_dstrategy(dp->d_rawdev, bio);
382 }
383 done:
384 if (reterr) {
385 if (msg->any.head.cmd & DMSGF_DELETE) {
386 if (iost && iost->count == 0) {
387 kfree(iost, M_DEVBUF);
388 msg->state->any.any = NULL;
389 }
390 kdmsg_msg_reply(msg, error);
391 } else {
392 kdmsg_msg_result(msg, error);
393 }
394 }
395 }
396
397 static
398 void
399 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg)
400 {
401 struct dios_io *iost;
402 struct buf *bp;
403 struct bio *bio;
404 int error = DMSG_ERR_NOSUPP;
405 int reterr = 1;
406
407 /*
408 * Only DMSG_BLK_WRITE commands imply read ops.
409 */
410 iost = msg->state->any.any;
411 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) {
412 if (msg->any.blk_write.bytes < DEV_BSIZE ||
413 msg->any.blk_write.bytes > MAXPHYS) {
414 error = DMSG_ERR_PARAM;
415 goto done;
416 }
417 if (iost == NULL) {
418 iost = kmalloc(sizeof(*iost), M_DEVBUF,
419 M_WAITOK | M_ZERO);
420 msg->state->any.any = iost;
421 }
422
423 /*
424 * Issue WRITE. Short data implies zeros. Try to optimize
425 * the buffer cache buffer for the case where we can just
426 * use the message's data pointer.
427 */
428 reterr = 0;
429 if (msg->aux_size >= msg->any.blk_write.bytes)
430 bp = getpbuf(NULL);
431 else
432 bp = geteblk(msg->any.blk_write.bytes);
433 bio = &bp->b_bio1;
434 bp->b_cmd = BUF_CMD_WRITE;
435 bp->b_bcount = msg->any.blk_write.bytes;
436 bp->b_resid = bp->b_bcount;
437 if (msg->aux_size >= msg->any.blk_write.bytes) {
438 bp->b_data = msg->aux_data;
439 kdmsg_detach_aux_data(msg, &iost->data);
440 } else {
441 bcopy(msg->aux_data, bp->b_data, msg->aux_size);
442 bzero(bp->b_data + msg->aux_size,
443 msg->any.blk_write.bytes - msg->aux_size);
444 bzero(&iost->data, sizeof(iost->data));
445 }
446 bio->bio_offset = msg->any.blk_write.offset;
447 bio->bio_caller_info1.ptr = msg->state;
448 bio->bio_done = diskiodone;
449
450 /* kdmsg_state_hold(msg->state); */
451 atomic_add_int(&blk_active, 1);
452 atomic_add_int(&iost->count, 1);
453 if (msg->any.head.cmd & DMSGF_DELETE)
454 iost->eof = 1;
455 BUF_KERNPROC(bp);
456 dev_dstrategy(dp->d_rawdev, bio);
457 }
458 done:
459 if (reterr) {
460 if (msg->any.head.cmd & DMSGF_DELETE) {
461 if (iost && iost->count == 0) {
462 kfree(iost, M_DEVBUF);
463 msg->state->any.any = NULL;
464 }
465 kdmsg_msg_reply(msg, error);
466 } else {
467 kdmsg_msg_result(msg, error);
468 }
469 }
470 }
471
472 static
473 void
474 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg)
475 {
476 struct dios_io *iost;
477 struct buf *bp;
478 struct bio *bio;
479 int error = DMSG_ERR_NOSUPP;
480 int reterr = 1;
481
482 /*
483 * Only DMSG_BLK_FLUSH commands imply read ops.
484 */
485 iost = msg->state->any.any;
486 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) {
487 if (iost == NULL) {
488 iost = kmalloc(sizeof(*iost), M_DEVBUF,
489 M_WAITOK | M_ZERO);
490 msg->state->any.any = iost;
491 }
492 reterr = 0;
493 bp = getpbuf(NULL);
494 bio = &bp->b_bio1;
495 bp->b_cmd = BUF_CMD_FLUSH;
496 bp->b_bcount = msg->any.blk_flush.bytes;
497 bp->b_resid = 0;
498 bio->bio_offset = msg->any.blk_flush.offset;
499 bio->bio_caller_info1.ptr = msg->state;
500 bio->bio_done = diskiodone;
501
502 /* kdmsg_state_hold(msg->state); */
503 atomic_add_int(&blk_active, 1);
504 atomic_add_int(&iost->count, 1);
505 if (msg->any.head.cmd & DMSGF_DELETE)
506 iost->eof = 1;
507 BUF_KERNPROC(bp);
508 dev_dstrategy(dp->d_rawdev, bio);
509 }
510 if (reterr) {
511 if (msg->any.head.cmd & DMSGF_DELETE) {
512 if (iost && iost->count == 0) {
513 kfree(iost, M_DEVBUF);
514 msg->state->any.any = NULL;
515 }
516 kdmsg_msg_reply(msg, error);
517 } else {
518 kdmsg_msg_result(msg, error);
519 }
520 }
521 }
522
523 static
524 void
525 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg)
526 {
527 struct dios_io *iost;
528 struct buf *bp;
529 struct bio *bio;
530 int error = DMSG_ERR_NOSUPP;
531 int reterr = 1;
532
533 /*
534 * Only DMSG_BLK_FREEBLKS commands imply read ops.
535 */
536 iost = msg->state->any.any;
537 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) {
538 if (iost == NULL) {
539 iost = kmalloc(sizeof(*iost), M_DEVBUF,
540 M_WAITOK | M_ZERO);
541 msg->state->any.any = iost;
542 }
543 reterr = 0;
544 bp = getpbuf(NULL);
545 bio = &bp->b_bio1;
546 bp->b_cmd = BUF_CMD_FREEBLKS;
547 bp->b_bcount = msg->any.blk_freeblks.bytes;
548 bp->b_resid = 0;
549 bio->bio_offset = msg->any.blk_freeblks.offset;
550 bio->bio_caller_info1.ptr = msg->state;
551 bio->bio_done = diskiodone;
552
553 /* kdmsg_state_hold(msg->state); */
554 atomic_add_int(&blk_active, 1);
555 atomic_add_int(&iost->count, 1);
556 if (msg->any.head.cmd & DMSGF_DELETE)
557 iost->eof = 1;
558 BUF_KERNPROC(bp);
559 dev_dstrategy(dp->d_rawdev, bio);
560 }
561 if (reterr) {
562 if (msg->any.head.cmd & DMSGF_DELETE) {
563 if (iost && iost->count == 0) {
564 kfree(iost, M_DEVBUF);
565 msg->state->any.any = NULL;
566 }
567 kdmsg_msg_reply(msg, error);
568 } else {
569 kdmsg_msg_result(msg, error);
570 }
571 }
572 }
573
574 static
575 void
576 diskiodone(struct bio *bio)
577 {
578 struct buf *bp = bio->bio_buf;
579 kdmsg_state_t *state = bio->bio_caller_info1.ptr;
580 kdmsg_msg_t *rmsg;
581 struct dios_io *iost = state->any.any;
582 int error;
583 int resid = 0;
584 int bytes;
585 uint32_t cmd;
586 void *data;
587
588 cmd = DMSG_LNK_ERROR;
589 data = NULL;
590 bytes = 0;
591
592 switch(bp->b_cmd) {
593 case BUF_CMD_READ:
594 cmd = DMSG_LNK_ERROR;
595 data = bp->b_data;
596 bytes = bp->b_bcount;
597 /* fall through */
598 case BUF_CMD_WRITE:
599 if (bp->b_flags & B_ERROR) {
600 error = bp->b_error;
601 } else {
602 error = 0;
603 resid = bp->b_resid;
604 }
605 kdmsg_free_aux_data(&iost->data);
606 break;
607 case BUF_CMD_FLUSH:
608 case BUF_CMD_FREEBLKS:
609 if (bp->b_flags & B_ERROR)
610 error = bp->b_error;
611 else
612 error = 0;
613 break;
614 default:
615 panic("diskiodone: Unknown bio cmd = %d\n",
616 bio->bio_buf->b_cmd);
617 error = 0; /* avoid compiler warning */
618 break; /* NOT REACHED */
619 }
620
621 /*
622 * Convert error to DMSG_ERR_* code.
623 */
624 if (error)
625 error = DMSG_ERR_IO;
626
627 /*
628 * Convert LNK_ERROR or BLK_ERROR if non-zero resid. READS will
629 * have already converted cmd to BLK_ERROR and set up data to return.
630 */
631 if (resid && cmd == DMSG_LNK_ERROR)
632 cmd = DMSG_BLK_ERROR;
633 /* XXX txcmd is delayed so this won't work for streaming */
634 if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */
635 cmd |= DMSGF_CREATE;
636 if (iost->eof) {
637 if (atomic_fetchadd_int(&iost->count, -1) == 1)
638 cmd |= DMSGF_DELETE;
639 } else {
640 atomic_add_int(&iost->count, -1);
641 }
642 atomic_add_int(&blk_active, -1);
643 cmd |= DMSGF_REPLY;
644
645 /*
646 * Allocate a basic or extended reply. Be careful not to populate
647 * extended header fields unless we allocated an extended reply.
648 */
649 rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0);
650 if (data) {
651 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT);
652 rmsg->aux_size = bytes;
653 rmsg->flags |= KDMSG_FLAG_AUXALLOC;
654 bcopy(data, rmsg->aux_data, bytes);
655 }
656 rmsg->any.blk_error.head.error = error;
657 if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR)
658 rmsg->any.blk_error.resid = resid;
659 bio->bio_caller_info1.ptr = NULL;
660 /* kdmsg_state_drop(state); */
661 kdmsg_msg_write(rmsg);
662 if (bp->b_flags & B_PAGING) {
663 relpbuf(bio->bio_buf, NULL);
664 } else {
665 bp->b_flags |= B_INVAL | B_AGE;
666 brelse(bp);
667 }
668 }
DragonFly BSD