?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
kernel - Incidental MPLOCK removal
[dragonfly.git] / sys / kern / subr_diskiocom.c
blob946c3608574f409dba837b2a0d3f5b6c3b04f634
1 /*
2 * Copyright (c) 2012 The DragonFly Project. All rights reserved.
3 *
4 * This code is derived from software contributed to The DragonFly Project
5 * by Matthew Dillon <dillon@backplane.com>
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in
15 * the documentation and/or other materials provided with the
16 * distribution.
17 * 3. Neither the name of The DragonFly Project nor the names of its
18 * contributors may be used to endorse or promote products derived
19 * from this software without specific, prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
24 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
25 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
26 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
27 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
28 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
29 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
30 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
31 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34 #include <sys/param.h>
35 #include <sys/systm.h>
36 #include <sys/kernel.h>
37 #include <sys/proc.h>
38 #include <sys/sysctl.h>
39 #include <sys/buf.h>
40 #include <sys/conf.h>
41 #include <sys/disklabel.h>
42 #include <sys/disklabel32.h>
43 #include <sys/disklabel64.h>
44 #include <sys/diskslice.h>
45 #include <sys/diskmbr.h>
46 #include <sys/disk.h>
47 #include <sys/malloc.h>
48 #include <sys/device.h>
49 #include <sys/devfs.h>
50 #include <sys/thread.h>
51 #include <sys/queue.h>
52 #include <sys/lock.h>
53 #include <sys/stat.h>
54 #include <sys/uuid.h>
55 #include <sys/dmsg.h>
56
57 #include <sys/buf2.h>
58 #include <sys/msgport2.h>
59 #include <sys/thread2.h>
60
61 struct dios_open {
62 int openrd;
63 int openwr;
64 };
65
66 struct dios_io {
67 int count;
68 int eof;
69 kdmsg_data_t data;
70 };
71
72 static MALLOC_DEFINE(M_DMSG_DISK, "dmsg_disk", "disk dmsg");
73
74 static int blk_active;
75 SYSCTL_INT(_debug, OID_AUTO, blk_active, CTLFLAG_RW, &blk_active, 0,
76 "Number of active iocom IOs");
77
78 static int disk_iocom_reconnect(struct disk *dp, struct file *fp);
79 static int disk_rcvdmsg(kdmsg_msg_t *msg);
80
81 static void disk_blk_open(struct disk *dp, kdmsg_msg_t *msg);
82 static void disk_blk_read(struct disk *dp, kdmsg_msg_t *msg);
83 static void disk_blk_write(struct disk *dp, kdmsg_msg_t *msg);
84 static void disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg);
85 static void disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg);
86 static void diskiodone(struct bio *bio);
87
88 void
89 disk_iocom_init(struct disk *dp)
90 {
91 kdmsg_iocom_init(&dp->d_iocom, dp,
92 KDMSG_IOCOMF_AUTOCONN |
93 KDMSG_IOCOMF_AUTORXSPAN |
94 KDMSG_IOCOMF_AUTOTXSPAN,
95 M_DMSG_DISK, disk_rcvdmsg);
96 }
97
98 void
99 disk_iocom_update(struct disk *dp)
100 {
101 }
102
103 void
104 disk_iocom_uninit(struct disk *dp)
105 {
106 kdmsg_iocom_uninit(&dp->d_iocom);
107 }
108
109 int
110 disk_iocom_ioctl(struct disk *dp, int cmd, void *data)
111 {
112 struct file *fp;
113 struct disk_ioc_recluster *recl;
114 int error;
115
116 switch(cmd) {
117 case DIOCRECLUSTER:
118 recl = data;
119 fp = holdfp(curproc->p_fd, recl->fd, -1);
120 if (fp) {
121 error = disk_iocom_reconnect(dp, fp);
122 } else {
123 error = EINVAL;
124 }
125 break;
126 default:
127 error = EOPNOTSUPP;
128 break;
129 }
130 return error;
131 }
132
133 static
134 int
135 disk_iocom_reconnect(struct disk *dp, struct file *fp)
136 {
137 char devname[64];
138
139 ksnprintf(devname, sizeof(devname), "%s%d",
140 dev_dname(dp->d_rawdev), dkunit(dp->d_rawdev));
141
142 kdmsg_iocom_reconnect(&dp->d_iocom, fp, devname);
143
144 dp->d_iocom.auto_lnk_conn.proto_version = DMSG_SPAN_PROTO_1;
145 dp->d_iocom.auto_lnk_conn.peer_type = DMSG_PEER_BLOCK;
146 dp->d_iocom.auto_lnk_conn.peer_mask = 1LLU << DMSG_PEER_BLOCK;
147 dp->d_iocom.auto_lnk_conn.peer_mask = (uint64_t)-1;
148 #if 0
149 if (dp->d_info.d_serialno) {
150 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
151 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
152 "%s/%s", hostname, dp->d_info.d_serialno);
153 } else {
154 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
155 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
156 "%s/%s", hostname, devname);
157 }
158 #endif
159 ksnprintf(dp->d_iocom.auto_lnk_conn.peer_label,
160 sizeof(dp->d_iocom.auto_lnk_conn.peer_label),
161 "%s/%s", hostname, devname);
162
163 dp->d_iocom.auto_lnk_span.proto_version = DMSG_SPAN_PROTO_1;
164 dp->d_iocom.auto_lnk_span.peer_type = DMSG_PEER_BLOCK;
165 dp->d_iocom.auto_lnk_span.media.block.bytes =
166 dp->d_info.d_media_size;
167 dp->d_iocom.auto_lnk_span.media.block.blksize =
168 dp->d_info.d_media_blksize;
169 ksnprintf(dp->d_iocom.auto_lnk_span.peer_label,
170 sizeof(dp->d_iocom.auto_lnk_span.peer_label),
171 "%s", dp->d_iocom.auto_lnk_conn.peer_label);
172 if (dp->d_info.d_serialno) {
173 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
174 sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
175 "%s", dp->d_info.d_serialno);
176 } else {
177 /*
178 * If no serial number is available generate a dummy serial
179 * number from the host and device name and pray. This will
180 * allow e.g. /dev/vn* to look meaningful on a remote machine.
181 */
182 ksnprintf(dp->d_iocom.auto_lnk_span.pfs_label,
183 sizeof(dp->d_iocom.auto_lnk_span.pfs_label),
184 "%s.%s", hostname, devname);
185 }
186
187 kdmsg_iocom_autoinitiate(&dp->d_iocom, NULL);
188
189 return (0);
190 }
191
192 static int
193 disk_rcvdmsg(kdmsg_msg_t *msg)
194 {
195 struct disk *dp = msg->state->iocom->handle;
196
197 /*
198 * Handle debug messages (these might not be in transactions)
199 */
200 switch(msg->any.head.cmd & DMSGF_CMDSWMASK) {
201 case DMSG_DBG_SHELL:
202 /*
203 * Execute shell command (not supported atm)
204 */
205 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
206 return(0);
207 case DMSG_DBG_SHELL | DMSGF_REPLY:
208 if (msg->aux_data) {
209 msg->aux_data[msg->aux_size - 1] = 0;
210 kprintf("diskiocom: DEBUGMSG: %s\n", msg->aux_data);
211 }
212 return(0);
213 }
214
215 /*
216 * All remaining messages must be in a transaction.
217 *
218 * NOTE! We currently don't care if the transaction is just
219 * the span transaction (for disk probes) or if it is the
220 * BLK_OPEN transaction.
221 *
222 * NOTE! We are switching on the first message's command. The
223 * actual message command within the transaction may be
224 * different (if streaming within a transaction).
225 */
226 if (msg->state == &msg->state->iocom->state0) {
227 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
228 return(0);
229 }
230
231 switch(msg->state->rxcmd & DMSGF_CMDSWMASK) {
232 case DMSG_BLK_OPEN:
233 disk_blk_open(dp, msg);
234 break;
235 case DMSG_BLK_READ:
236 /*
237 * not reached normally but leave in for completeness
238 */
239 disk_blk_read(dp, msg);
240 break;
241 case DMSG_BLK_WRITE:
242 disk_blk_write(dp, msg);
243 break;
244 case DMSG_BLK_FLUSH:
245 disk_blk_flush(dp, msg);
246 break;
247 case DMSG_BLK_FREEBLKS:
248 disk_blk_freeblks(dp, msg);
249 break;
250 default:
251 if ((msg->any.head.cmd & DMSGF_REPLY) == 0) {
252 if (msg->any.head.cmd & DMSGF_DELETE)
253 kdmsg_msg_reply(msg, DMSG_ERR_NOSUPP);
254 else
255 kdmsg_msg_result(msg, DMSG_ERR_NOSUPP);
256 }
257 break;
258 }
259 return (0);
260 }
261
262 static
263 void
264 disk_blk_open(struct disk *dp, kdmsg_msg_t *msg)
265 {
266 struct dios_open *openst;
267 int error = DMSG_ERR_NOSUPP;
268 int fflags;
269
270 openst = msg->state->any.any;
271 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_OPEN) {
272 if (openst == NULL) {
273 openst = kmalloc(sizeof(*openst), M_DEVBUF,
274 M_WAITOK | M_ZERO);
275 msg->state->any.any = openst;
276 }
277 fflags = 0;
278 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
279 fflags = FREAD;
280 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
281 fflags |= FWRITE;
282 error = dev_dopen(dp->d_rawdev, fflags, S_IFCHR, proc0.p_ucred, NULL);
283 if (error) {
284 error = DMSG_ERR_IO;
285 } else {
286 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
287 ++openst->openrd;
288 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
289 ++openst->openwr;
290 }
291 }
292 #if 0
293 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_CLOSE &&
294 openst) {
295 fflags = 0;
296 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_RD) &&
297 openst->openrd) {
298 fflags = FREAD;
299 }
300 if ((msg->any.blk_open.modes & DMSG_BLKOPEN_WR) &&
301 openst->openwr) {
302 fflags |= FWRITE;
303 }
304 error = dev_dclose(dp->d_rawdev, fflags, S_IFCHR, NULL);
305 if (error) {
306 error = DMSG_ERR_IO;
307 } else {
308 if (msg->any.blk_open.modes & DMSG_BLKOPEN_RD)
309 --openst->openrd;
310 if (msg->any.blk_open.modes & DMSG_BLKOPEN_WR)
311 --openst->openwr;
312 }
313 }
314 #endif
315 if (msg->any.head.cmd & DMSGF_DELETE) {
316 if (openst) {
317 while (openst->openrd && openst->openwr) {
318 --openst->openrd;
319 --openst->openwr;
320 dev_dclose(dp->d_rawdev, FREAD|FWRITE, S_IFCHR, NULL);
321 }
322 while (openst->openrd) {
323 --openst->openrd;
324 dev_dclose(dp->d_rawdev, FREAD, S_IFCHR, NULL);
325 }
326 while (openst->openwr) {
327 --openst->openwr;
328 dev_dclose(dp->d_rawdev, FWRITE, S_IFCHR, NULL);
329 }
330 kfree(openst, M_DEVBUF);
331 msg->state->any.any = NULL;
332 }
333 kdmsg_msg_reply(msg, error);
334 } else {
335 kdmsg_msg_result(msg, error);
336 }
337 }
338
339 static
340 void
341 disk_blk_read(struct disk *dp, kdmsg_msg_t *msg)
342 {
343 struct dios_io *iost;
344 struct buf *bp;
345 struct bio *bio;
346 int error = DMSG_ERR_NOSUPP;
347 int reterr = 1;
348
349 /*
350 * Only DMSG_BLK_READ commands imply read ops.
351 */
352 iost = msg->state->any.any;
353 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_READ) {
354 if (msg->any.blk_read.bytes < DEV_BSIZE ||
355 msg->any.blk_read.bytes > MAXPHYS) {
356 error = DMSG_ERR_PARAM;
357 goto done;
358 }
359 if (iost == NULL) {
360 iost = kmalloc(sizeof(*iost), M_DEVBUF,
361 M_WAITOK | M_ZERO);
362 msg->state->any.any = iost;
363 }
364 reterr = 0;
365 bp = geteblk(msg->any.blk_read.bytes);
366 bio = &bp->b_bio1;
367 bp->b_cmd = BUF_CMD_READ;
368 bp->b_bcount = msg->any.blk_read.bytes;
369 bp->b_resid = bp->b_bcount;
370 bio->bio_offset = msg->any.blk_read.offset;
371 bio->bio_caller_info1.ptr = msg->state;
372 bio->bio_done = diskiodone;
373
374 /* kdmsg_state_hold(msg->state); */
375 atomic_add_int(&blk_active, 1);
376 atomic_add_int(&iost->count, 1);
377 if (msg->any.head.cmd & DMSGF_DELETE)
378 iost->eof = 1;
379 BUF_KERNPROC(bp);
380 dev_dstrategy(dp->d_rawdev, bio);
381 }
382 done:
383 if (reterr) {
384 if (msg->any.head.cmd & DMSGF_DELETE) {
385 if (iost && iost->count == 0) {
386 kfree(iost, M_DEVBUF);
387 msg->state->any.any = NULL;
388 }
389 kdmsg_msg_reply(msg, error);
390 } else {
391 kdmsg_msg_result(msg, error);
392 }
393 }
394 }
395
396 static
397 void
398 disk_blk_write(struct disk *dp, kdmsg_msg_t *msg)
399 {
400 struct dios_io *iost;
401 struct buf *bp;
402 struct bio *bio;
403 int error = DMSG_ERR_NOSUPP;
404 int reterr = 1;
405
406 /*
407 * Only DMSG_BLK_WRITE commands imply read ops.
408 */
409 iost = msg->state->any.any;
410 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_WRITE) {
411 if (msg->any.blk_write.bytes < DEV_BSIZE ||
412 msg->any.blk_write.bytes > MAXPHYS) {
413 error = DMSG_ERR_PARAM;
414 goto done;
415 }
416 if (iost == NULL) {
417 iost = kmalloc(sizeof(*iost), M_DEVBUF,
418 M_WAITOK | M_ZERO);
419 msg->state->any.any = iost;
420 }
421
422 /*
423 * Issue WRITE. Short data implies zeros. Try to optimize
424 * the buffer cache buffer for the case where we can just
425 * use the message's data pointer.
426 */
427 reterr = 0;
428 if (msg->aux_size >= msg->any.blk_write.bytes)
429 bp = getpbuf(NULL);
430 else
431 bp = geteblk(msg->any.blk_write.bytes);
432 bio = &bp->b_bio1;
433 bp->b_cmd = BUF_CMD_WRITE;
434 bp->b_bcount = msg->any.blk_write.bytes;
435 bp->b_resid = bp->b_bcount;
436 if (msg->aux_size >= msg->any.blk_write.bytes) {
437 bp->b_data = msg->aux_data;
438 kdmsg_detach_aux_data(msg, &iost->data);
439 } else {
440 bcopy(msg->aux_data, bp->b_data, msg->aux_size);
441 bzero(bp->b_data + msg->aux_size,
442 msg->any.blk_write.bytes - msg->aux_size);
443 bzero(&iost->data, sizeof(iost->data));
444 }
445 bio->bio_offset = msg->any.blk_write.offset;
446 bio->bio_caller_info1.ptr = msg->state;
447 bio->bio_done = diskiodone;
448
449 /* kdmsg_state_hold(msg->state); */
450 atomic_add_int(&blk_active, 1);
451 atomic_add_int(&iost->count, 1);
452 if (msg->any.head.cmd & DMSGF_DELETE)
453 iost->eof = 1;
454 BUF_KERNPROC(bp);
455 dev_dstrategy(dp->d_rawdev, bio);
456 }
457 done:
458 if (reterr) {
459 if (msg->any.head.cmd & DMSGF_DELETE) {
460 if (iost && iost->count == 0) {
461 kfree(iost, M_DEVBUF);
462 msg->state->any.any = NULL;
463 }
464 kdmsg_msg_reply(msg, error);
465 } else {
466 kdmsg_msg_result(msg, error);
467 }
468 }
469 }
470
471 static
472 void
473 disk_blk_flush(struct disk *dp, kdmsg_msg_t *msg)
474 {
475 struct dios_io *iost;
476 struct buf *bp;
477 struct bio *bio;
478 int error = DMSG_ERR_NOSUPP;
479 int reterr = 1;
480
481 /*
482 * Only DMSG_BLK_FLUSH commands imply read ops.
483 */
484 iost = msg->state->any.any;
485 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FLUSH) {
486 if (iost == NULL) {
487 iost = kmalloc(sizeof(*iost), M_DEVBUF,
488 M_WAITOK | M_ZERO);
489 msg->state->any.any = iost;
490 }
491 reterr = 0;
492 bp = getpbuf(NULL);
493 bio = &bp->b_bio1;
494 bp->b_cmd = BUF_CMD_FLUSH;
495 bp->b_bcount = msg->any.blk_flush.bytes;
496 bp->b_resid = 0;
497 bio->bio_offset = msg->any.blk_flush.offset;
498 bio->bio_caller_info1.ptr = msg->state;
499 bio->bio_done = diskiodone;
500
501 /* kdmsg_state_hold(msg->state); */
502 atomic_add_int(&blk_active, 1);
503 atomic_add_int(&iost->count, 1);
504 if (msg->any.head.cmd & DMSGF_DELETE)
505 iost->eof = 1;
506 BUF_KERNPROC(bp);
507 dev_dstrategy(dp->d_rawdev, bio);
508 }
509 if (reterr) {
510 if (msg->any.head.cmd & DMSGF_DELETE) {
511 if (iost && iost->count == 0) {
512 kfree(iost, M_DEVBUF);
513 msg->state->any.any = NULL;
514 }
515 kdmsg_msg_reply(msg, error);
516 } else {
517 kdmsg_msg_result(msg, error);
518 }
519 }
520 }
521
522 static
523 void
524 disk_blk_freeblks(struct disk *dp, kdmsg_msg_t *msg)
525 {
526 struct dios_io *iost;
527 struct buf *bp;
528 struct bio *bio;
529 int error = DMSG_ERR_NOSUPP;
530 int reterr = 1;
531
532 /*
533 * Only DMSG_BLK_FREEBLKS commands imply read ops.
534 */
535 iost = msg->state->any.any;
536 if ((msg->any.head.cmd & DMSGF_CMDSWMASK) == DMSG_BLK_FREEBLKS) {
537 if (iost == NULL) {
538 iost = kmalloc(sizeof(*iost), M_DEVBUF,
539 M_WAITOK | M_ZERO);
540 msg->state->any.any = iost;
541 }
542 reterr = 0;
543 bp = getpbuf(NULL);
544 bio = &bp->b_bio1;
545 bp->b_cmd = BUF_CMD_FREEBLKS;
546 bp->b_bcount = msg->any.blk_freeblks.bytes;
547 bp->b_resid = 0;
548 bio->bio_offset = msg->any.blk_freeblks.offset;
549 bio->bio_caller_info1.ptr = msg->state;
550 bio->bio_done = diskiodone;
551
552 /* kdmsg_state_hold(msg->state); */
553 atomic_add_int(&blk_active, 1);
554 atomic_add_int(&iost->count, 1);
555 if (msg->any.head.cmd & DMSGF_DELETE)
556 iost->eof = 1;
557 BUF_KERNPROC(bp);
558 dev_dstrategy(dp->d_rawdev, bio);
559 }
560 if (reterr) {
561 if (msg->any.head.cmd & DMSGF_DELETE) {
562 if (iost && iost->count == 0) {
563 kfree(iost, M_DEVBUF);
564 msg->state->any.any = NULL;
565 }
566 kdmsg_msg_reply(msg, error);
567 } else {
568 kdmsg_msg_result(msg, error);
569 }
570 }
571 }
572
573 static
574 void
575 diskiodone(struct bio *bio)
576 {
577 struct buf *bp = bio->bio_buf;
578 kdmsg_state_t *state = bio->bio_caller_info1.ptr;
579 kdmsg_msg_t *rmsg;
580 struct dios_io *iost = state->any.any;
581 int error;
582 int resid = 0;
583 int bytes;
584 uint32_t cmd;
585 void *data;
586
587 cmd = DMSG_LNK_ERROR;
588 data = NULL;
589 bytes = 0;
590
591 switch(bp->b_cmd) {
592 case BUF_CMD_READ:
593 cmd = DMSG_LNK_ERROR;
594 data = bp->b_data;
595 bytes = bp->b_bcount;
596 /* fall through */
597 case BUF_CMD_WRITE:
598 if (bp->b_flags & B_ERROR) {
599 error = bp->b_error;
600 } else {
601 error = 0;
602 resid = bp->b_resid;
603 }
604 kdmsg_free_aux_data(&iost->data);
605 break;
606 case BUF_CMD_FLUSH:
607 case BUF_CMD_FREEBLKS:
608 if (bp->b_flags & B_ERROR)
609 error = bp->b_error;
610 else
611 error = 0;
612 break;
613 default:
614 panic("diskiodone: Unknown bio cmd = %d\n",
615 bio->bio_buf->b_cmd);
616 error = 0; /* avoid compiler warning */
617 break; /* NOT REACHED */
618 }
619
620 /*
621 * Convert error to DMSG_ERR_* code.
622 */
623 if (error)
624 error = DMSG_ERR_IO;
625
626 /*
627 * Convert LNK_ERROR or BLK_ERROR if non-zero resid. READS will
628 * have already converted cmd to BLK_ERROR and set up data to return.
629 */
630 if (resid && cmd == DMSG_LNK_ERROR)
631 cmd = DMSG_BLK_ERROR;
632 /* XXX txcmd is delayed so this won't work for streaming */
633 if ((state->txcmd & DMSGF_CREATE) == 0) /* assume serialized */
634 cmd |= DMSGF_CREATE;
635 if (iost->eof) {
636 if (atomic_fetchadd_int(&iost->count, -1) == 1)
637 cmd |= DMSGF_DELETE;
638 } else {
639 atomic_add_int(&iost->count, -1);
640 }
641 atomic_add_int(&blk_active, -1);
642 cmd |= DMSGF_REPLY;
643
644 /*
645 * Allocate a basic or extended reply. Be careful not to populate
646 * extended header fields unless we allocated an extended reply.
647 */
648 rmsg = kdmsg_msg_alloc(state, cmd, NULL, 0);
649 if (data) {
650 rmsg->aux_data = kmalloc(bytes, state->iocom->mmsg, M_INTWAIT);
651 rmsg->aux_size = bytes;
652 rmsg->flags |= KDMSG_FLAG_AUXALLOC;
653 bcopy(data, rmsg->aux_data, bytes);
654 }
655 rmsg->any.blk_error.head.error = error;
656 if ((cmd & DMSGF_BASECMDMASK) == DMSG_BLK_ERROR)
657 rmsg->any.blk_error.resid = resid;
658 bio->bio_caller_info1.ptr = NULL;
659 /* kdmsg_state_drop(state); */
660 kdmsg_msg_write(rmsg);
661 if (bp->b_flags & B_PAGING) {
662 relpbuf(bio->bio_buf, NULL);
663 } else {
664 bp->b_flags |= B_INVAL | B_AGE;
665 brelse(bp);
666 }
667 }
DragonFly BSD