?  search:
summary | log | graphiclog1 | graphiclog2 | commit | commitdiff | tree | refs | edit | fork
blame | history | raw | HEAD
6305 blkdev should provide the device_error kstat for iostat -E
[unleashed.git] / usr / src / uts / common / io / blkdev / blkdev.c
blob7534139884d67fc6b6f0252f6d9e608519221a95
1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2009, 2010, Oracle and/or its affiliates. All rights reserved.
23 * Copyright 2012 Garrett D'Amore <garrett@damore.org>. All rights reserved.
24 * Copyright 2012 Alexey Zaytsev <alexey.zaytsev@gmail.com> All rights reserved.
25 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
26 */
27
28 #include <sys/types.h>
29 #include <sys/ksynch.h>
30 #include <sys/kmem.h>
31 #include <sys/file.h>
32 #include <sys/errno.h>
33 #include <sys/open.h>
34 #include <sys/buf.h>
35 #include <sys/uio.h>
36 #include <sys/aio_req.h>
37 #include <sys/cred.h>
38 #include <sys/modctl.h>
39 #include <sys/cmlb.h>
40 #include <sys/conf.h>
41 #include <sys/devops.h>
42 #include <sys/list.h>
43 #include <sys/sysmacros.h>
44 #include <sys/dkio.h>
45 #include <sys/vtoc.h>
46 #include <sys/scsi/scsi.h> /* for DTYPE_DIRECT */
47 #include <sys/kstat.h>
48 #include <sys/fs/dv_node.h>
49 #include <sys/ddi.h>
50 #include <sys/sunddi.h>
51 #include <sys/note.h>
52 #include <sys/blkdev.h>
53 #include <sys/scsi/impl/inquiry.h>
54
55 #define BD_MAXPART 64
56 #define BDINST(dev) (getminor(dev) / BD_MAXPART)
57 #define BDPART(dev) (getminor(dev) % BD_MAXPART)
58
59 typedef struct bd bd_t;
60 typedef struct bd_xfer_impl bd_xfer_impl_t;
61
62 struct bd {
63 void *d_private;
64 dev_info_t *d_dip;
65 kmutex_t d_ocmutex;
66 kmutex_t d_iomutex;
67 kmutex_t *d_errmutex;
68 kmutex_t d_statemutex;
69 kcondvar_t d_statecv;
70 enum dkio_state d_state;
71 cmlb_handle_t d_cmlbh;
72 unsigned d_open_lyr[BD_MAXPART]; /* open count */
73 uint64_t d_open_excl; /* bit mask indexed by partition */
74 uint64_t d_open_reg[OTYPCNT]; /* bit mask */
75
76 uint32_t d_qsize;
77 uint32_t d_qactive;
78 uint32_t d_maxxfer;
79 uint32_t d_blkshift;
80 uint32_t d_pblkshift;
81 uint64_t d_numblks;
82 ddi_devid_t d_devid;
83
84 kmem_cache_t *d_cache;
85 list_t d_runq;
86 list_t d_waitq;
87 kstat_t *d_ksp;
88 kstat_io_t *d_kiop;
89 kstat_t *d_errstats;
90 struct bd_errstats *d_kerr;
91
92 boolean_t d_rdonly;
93 boolean_t d_ssd;
94 boolean_t d_removable;
95 boolean_t d_hotpluggable;
96 boolean_t d_use_dma;
97
98 ddi_dma_attr_t d_dma;
99 bd_ops_t d_ops;
100 bd_handle_t d_handle;
101 };
102
103 struct bd_handle {
104 bd_ops_t h_ops;
105 ddi_dma_attr_t *h_dma;
106 dev_info_t *h_parent;
107 dev_info_t *h_child;
108 void *h_private;
109 bd_t *h_bd;
110 char *h_name;
111 char h_addr[20]; /* enough for %X,%X */
112 };
113
114 struct bd_xfer_impl {
115 bd_xfer_t i_public;
116 list_node_t i_linkage;
117 bd_t *i_bd;
118 buf_t *i_bp;
119 uint_t i_num_win;
120 uint_t i_cur_win;
121 off_t i_offset;
122 int (*i_func)(void *, bd_xfer_t *);
123 uint32_t i_blkshift;
124 size_t i_len;
125 size_t i_resid;
126 };
127
128 #define i_dmah i_public.x_dmah
129 #define i_dmac i_public.x_dmac
130 #define i_ndmac i_public.x_ndmac
131 #define i_kaddr i_public.x_kaddr
132 #define i_nblks i_public.x_nblks
133 #define i_blkno i_public.x_blkno
134 #define i_flags i_public.x_flags
135
136
137 /*
138 * Private prototypes.
139 */
140
141 static void bd_prop_update_inqstring(dev_info_t *, char *, char *, size_t);
142 static void bd_create_inquiry_props(dev_info_t *, bd_drive_t *);
143 static void bd_create_errstats(bd_t *, int, bd_drive_t *);
144 static void bd_errstats_setstr(kstat_named_t *, char *, size_t, char *);
145 static void bd_init_errstats(bd_t *, bd_drive_t *);
146
147 static int bd_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
148 static int bd_attach(dev_info_t *, ddi_attach_cmd_t);
149 static int bd_detach(dev_info_t *, ddi_detach_cmd_t);
150
151 static int bd_open(dev_t *, int, int, cred_t *);
152 static int bd_close(dev_t, int, int, cred_t *);
153 static int bd_strategy(struct buf *);
154 static int bd_ioctl(dev_t, int, intptr_t, int, cred_t *, int *);
155 static int bd_dump(dev_t, caddr_t, daddr_t, int);
156 static int bd_read(dev_t, struct uio *, cred_t *);
157 static int bd_write(dev_t, struct uio *, cred_t *);
158 static int bd_aread(dev_t, struct aio_req *, cred_t *);
159 static int bd_awrite(dev_t, struct aio_req *, cred_t *);
160 static int bd_prop_op(dev_t, dev_info_t *, ddi_prop_op_t, int, char *,
161 caddr_t, int *);
162
163 static int bd_tg_rdwr(dev_info_t *, uchar_t, void *, diskaddr_t, size_t,
164 void *);
165 static int bd_tg_getinfo(dev_info_t *, int, void *, void *);
166 static int bd_xfer_ctor(void *, void *, int);
167 static void bd_xfer_dtor(void *, void *);
168 static void bd_sched(bd_t *);
169 static void bd_submit(bd_t *, bd_xfer_impl_t *);
170 static void bd_runq_exit(bd_xfer_impl_t *, int);
171 static void bd_update_state(bd_t *);
172 static int bd_check_state(bd_t *, enum dkio_state *);
173 static int bd_flush_write_cache(bd_t *, struct dk_callback *);
174
175 struct cmlb_tg_ops bd_tg_ops = {
176 TG_DK_OPS_VERSION_1,
177 bd_tg_rdwr,
178 bd_tg_getinfo,
179 };
180
181 static struct cb_ops bd_cb_ops = {
182 bd_open, /* open */
183 bd_close, /* close */
184 bd_strategy, /* strategy */
185 nodev, /* print */
186 bd_dump, /* dump */
187 bd_read, /* read */
188 bd_write, /* write */
189 bd_ioctl, /* ioctl */
190 nodev, /* devmap */
191 nodev, /* mmap */
192 nodev, /* segmap */
193 nochpoll, /* poll */
194 bd_prop_op, /* cb_prop_op */
195 0, /* streamtab */
196 D_64BIT | D_MP, /* Driver comaptibility flag */
197 CB_REV, /* cb_rev */
198 bd_aread, /* async read */
199 bd_awrite /* async write */
200 };
201
202 struct dev_ops bd_dev_ops = {
203 DEVO_REV, /* devo_rev, */
204 0, /* refcnt */
205 bd_getinfo, /* getinfo */
206 nulldev, /* identify */
207 nulldev, /* probe */
208 bd_attach, /* attach */
209 bd_detach, /* detach */
210 nodev, /* reset */
211 &bd_cb_ops, /* driver operations */
212 NULL, /* bus operations */
213 NULL, /* power */
214 ddi_quiesce_not_needed, /* quiesce */
215 };
216
217 static struct modldrv modldrv = {
218 &mod_driverops,
219 "Generic Block Device",
220 &bd_dev_ops,
221 };
222
223 static struct modlinkage modlinkage = {
224 MODREV_1, { &modldrv, NULL }
225 };
226
227 static void *bd_state;
228 static krwlock_t bd_lock;
229
230 int
231 _init(void)
232 {
233 int rv;
234
235 rv = ddi_soft_state_init(&bd_state, sizeof (struct bd), 2);
236 if (rv != DDI_SUCCESS) {
237 return (rv);
238 }
239 rw_init(&bd_lock, NULL, RW_DRIVER, NULL);
240 rv = mod_install(&modlinkage);
241 if (rv != DDI_SUCCESS) {
242 rw_destroy(&bd_lock);
243 ddi_soft_state_fini(&bd_state);
244 }
245 return (rv);
246 }
247
248 int
249 _fini(void)
250 {
251 int rv;
252
253 rv = mod_remove(&modlinkage);
254 if (rv == DDI_SUCCESS) {
255 rw_destroy(&bd_lock);
256 ddi_soft_state_fini(&bd_state);
257 }
258 return (rv);
259 }
260
261 int
262 _info(struct modinfo *modinfop)
263 {
264 return (mod_info(&modlinkage, modinfop));
265 }
266
267 static int
268 bd_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg, void **resultp)
269 {
270 bd_t *bd;
271 minor_t inst;
272
273 _NOTE(ARGUNUSED(dip));
274
275 inst = BDINST((dev_t)arg);
276
277 switch (cmd) {
278 case DDI_INFO_DEVT2DEVINFO:
279 bd = ddi_get_soft_state(bd_state, inst);
280 if (bd == NULL) {
281 return (DDI_FAILURE);
282 }
283 *resultp = (void *)bd->d_dip;
284 break;
285
286 case DDI_INFO_DEVT2INSTANCE:
287 *resultp = (void *)(intptr_t)inst;
288 break;
289
290 default:
291 return (DDI_FAILURE);
292 }
293 return (DDI_SUCCESS);
294 }
295
296 static void
297 bd_prop_update_inqstring(dev_info_t *dip, char *name, char *data, size_t len)
298 {
299 int ilen;
300 char *data_string;
301
302 ilen = scsi_ascii_inquiry_len(data, len);
303 ASSERT3U(ilen, <=, len);
304 if (ilen <= 0)
305 return;
306 /* ensure null termination */
307 data_string = kmem_zalloc(ilen + 1, KM_SLEEP);
308 bcopy(data, data_string, ilen);
309 (void) ndi_prop_update_string(DDI_DEV_T_NONE, dip, name, data_string);
310 kmem_free(data_string, ilen + 1);
311 }
312
313 static void
314 bd_create_inquiry_props(dev_info_t *dip, bd_drive_t *drive)
315 {
316 if (drive->d_vendor_len > 0)
317 bd_prop_update_inqstring(dip, INQUIRY_VENDOR_ID,
318 drive->d_vendor, drive->d_vendor_len);
319
320 if (drive->d_product_len > 0)
321 bd_prop_update_inqstring(dip, INQUIRY_PRODUCT_ID,
322 drive->d_product, drive->d_product_len);
323
324 if (drive->d_serial_len > 0)
325 bd_prop_update_inqstring(dip, INQUIRY_SERIAL_NO,
326 drive->d_serial, drive->d_serial_len);
327
328 if (drive->d_revision_len > 0)
329 bd_prop_update_inqstring(dip, INQUIRY_REVISION_ID,
330 drive->d_revision, drive->d_revision_len);
331 }
332
333 static void
334 bd_create_errstats(bd_t *bd, int inst, bd_drive_t *drive)
335 {
336 char ks_module[KSTAT_STRLEN];
337 char ks_name[KSTAT_STRLEN];
338 int ndata = sizeof (struct bd_errstats) / sizeof (kstat_named_t);
339
340 if (bd->d_errstats != NULL)
341 return;
342
343 (void) snprintf(ks_module, sizeof (ks_module), "%serr",
344 ddi_driver_name(bd->d_dip));
345 (void) snprintf(ks_name, sizeof (ks_name), "%s%d,err",
346 ddi_driver_name(bd->d_dip), inst);
347
348 bd->d_errstats = kstat_create(ks_module, inst, ks_name, "device_error",
349 KSTAT_TYPE_NAMED, ndata, KSTAT_FLAG_PERSISTENT);
350
351 if (bd->d_errstats == NULL) {
352 /*
353 * Even if we cannot create the kstat, we create a
354 * scratch kstat. The reason for this is to ensure
355 * that we can update the kstat all of the time,
356 * without adding an extra branch instruction.
357 */
358 bd->d_kerr = kmem_zalloc(sizeof (struct bd_errstats),
359 KM_SLEEP);
360 bd->d_errmutex = kmem_zalloc(sizeof (kmutex_t), KM_SLEEP);
361 mutex_init(bd->d_errmutex, NULL, MUTEX_DRIVER, NULL);
362 } else {
363 if (bd->d_errstats->ks_lock == NULL) {
364 bd->d_errstats->ks_lock = kmem_zalloc(sizeof (kmutex_t),
365 KM_SLEEP);
366 mutex_init(bd->d_errstats->ks_lock, NULL, MUTEX_DRIVER,
367 NULL);
368 }
369
370 bd->d_errmutex = bd->d_errstats->ks_lock;
371 bd->d_kerr = (struct bd_errstats *)bd->d_errstats->ks_data;
372 }
373
374 kstat_named_init(&bd->d_kerr->bd_softerrs, "Soft Errors",
375 KSTAT_DATA_UINT32);
376 kstat_named_init(&bd->d_kerr->bd_harderrs, "Hard Errors",
377 KSTAT_DATA_UINT32);
378 kstat_named_init(&bd->d_kerr->bd_transerrs, "Transport Errors",
379 KSTAT_DATA_UINT32);
380
381 if (drive->d_model_len > 0) {
382 kstat_named_init(&bd->d_kerr->bd_model, "Model",
383 KSTAT_DATA_STRING);
384 } else {
385 kstat_named_init(&bd->d_kerr->bd_vid, "Vendor",
386 KSTAT_DATA_STRING);
387 kstat_named_init(&bd->d_kerr->bd_pid, "Product",
388 KSTAT_DATA_STRING);
389 }
390
391 kstat_named_init(&bd->d_kerr->bd_revision, "Revision",
392 KSTAT_DATA_STRING);
393 kstat_named_init(&bd->d_kerr->bd_serial, "Serial No",
394 KSTAT_DATA_STRING);
395 kstat_named_init(&bd->d_kerr->bd_capacity, "Size",
396 KSTAT_DATA_ULONGLONG);
397 kstat_named_init(&bd->d_kerr->bd_rq_media_err, "Media Error",
398 KSTAT_DATA_UINT32);
399 kstat_named_init(&bd->d_kerr->bd_rq_ntrdy_err, "Device Not Ready",
400 KSTAT_DATA_UINT32);
401 kstat_named_init(&bd->d_kerr->bd_rq_nodev_err, "No Device",
402 KSTAT_DATA_UINT32);
403 kstat_named_init(&bd->d_kerr->bd_rq_recov_err, "Recoverable",
404 KSTAT_DATA_UINT32);
405 kstat_named_init(&bd->d_kerr->bd_rq_illrq_err, "Illegal Request",
406 KSTAT_DATA_UINT32);
407 kstat_named_init(&bd->d_kerr->bd_rq_pfa_err,
408 "Predictive Failure Analysis", KSTAT_DATA_UINT32);
409
410 bd->d_errstats->ks_private = bd;
411
412 kstat_install(bd->d_errstats);
413 }
414
415 static void
416 bd_errstats_setstr(kstat_named_t *k, char *str, size_t len, char *alt)
417 {
418 char *tmp;
419
420 if (KSTAT_NAMED_STR_PTR(k) == NULL) {
421 if (len > 0) {
422 tmp = kmem_alloc(len + 1, KM_SLEEP);
423 (void) strlcpy(tmp, str, len);
424 } else {
425 tmp = alt;
426 }
427
428 kstat_named_setstr(k, tmp);
429 }
430 }
431
432 static void
433 bd_init_errstats(bd_t *bd, bd_drive_t *drive)
434 {
435 struct bd_errstats *est = bd->d_kerr;
436
437 mutex_enter(bd->d_errmutex);
438
439 if (drive->d_model_len > 0 &&
440 KSTAT_NAMED_STR_PTR(&est->bd_model) == NULL) {
441 bd_errstats_setstr(&est->bd_model, drive->d_model,
442 drive->d_model_len, NULL);
443 } else {
444 bd_errstats_setstr(&est->bd_vid, drive->d_vendor,
445 drive->d_vendor_len, "Unknown ");
446 bd_errstats_setstr(&est->bd_pid, drive->d_product,
447 drive->d_product_len, "Unknown ");
448 }
449
450 bd_errstats_setstr(&est->bd_revision, drive->d_revision,
451 drive->d_revision_len, "0001");
452 bd_errstats_setstr(&est->bd_serial, drive->d_serial,
453 drive->d_serial_len, "0 ");
454
455 mutex_exit(bd->d_errmutex);
456 }
457
458 static int
459 bd_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
460 {
461 int inst;
462 bd_handle_t hdl;
463 bd_t *bd;
464 bd_drive_t drive;
465 int rv;
466 char name[16];
467 char kcache[32];
468
469 switch (cmd) {
470 case DDI_ATTACH:
471 break;
472 case DDI_RESUME:
473 /* We don't do anything native for suspend/resume */
474 return (DDI_SUCCESS);
475 default:
476 return (DDI_FAILURE);
477 }
478
479 inst = ddi_get_instance(dip);
480 hdl = ddi_get_parent_data(dip);
481
482 (void) snprintf(name, sizeof (name), "%s%d",
483 ddi_driver_name(dip), ddi_get_instance(dip));
484 (void) snprintf(kcache, sizeof (kcache), "%s_xfer", name);
485
486 if (hdl == NULL) {
487 cmn_err(CE_WARN, "%s: missing parent data!", name);
488 return (DDI_FAILURE);
489 }
490
491 if (ddi_soft_state_zalloc(bd_state, inst) != DDI_SUCCESS) {
492 cmn_err(CE_WARN, "%s: unable to zalloc soft state!", name);
493 return (DDI_FAILURE);
494 }
495 bd = ddi_get_soft_state(bd_state, inst);
496
497 if (hdl->h_dma) {
498 bd->d_dma = *(hdl->h_dma);
499 bd->d_dma.dma_attr_granular =
500 max(DEV_BSIZE, bd->d_dma.dma_attr_granular);
501 bd->d_use_dma = B_TRUE;
502
503 if (bd->d_maxxfer &&
504 (bd->d_maxxfer != bd->d_dma.dma_attr_maxxfer)) {
505 cmn_err(CE_WARN,
506 "%s: inconsistent maximum transfer size!",
507 name);
508 /* We force it */
509 bd->d_maxxfer = bd->d_dma.dma_attr_maxxfer;
510 } else {
511 bd->d_maxxfer = bd->d_dma.dma_attr_maxxfer;
512 }
513 } else {
514 bd->d_use_dma = B_FALSE;
515 if (bd->d_maxxfer == 0) {
516 bd->d_maxxfer = 1024 * 1024;
517 }
518 }
519 bd->d_ops = hdl->h_ops;
520 bd->d_private = hdl->h_private;
521 bd->d_blkshift = 9; /* 512 bytes, to start */
522
523 if (bd->d_maxxfer % DEV_BSIZE) {
524 cmn_err(CE_WARN, "%s: maximum transfer misaligned!", name);
525 bd->d_maxxfer &= ~(DEV_BSIZE - 1);
526 }
527 if (bd->d_maxxfer < DEV_BSIZE) {
528 cmn_err(CE_WARN, "%s: maximum transfer size too small!", name);
529 ddi_soft_state_free(bd_state, inst);
530 return (DDI_FAILURE);
531 }
532
533 bd->d_dip = dip;
534 bd->d_handle = hdl;
535 hdl->h_bd = bd;
536 ddi_set_driver_private(dip, bd);
537
538 mutex_init(&bd->d_iomutex, NULL, MUTEX_DRIVER, NULL);
539 mutex_init(&bd->d_ocmutex, NULL, MUTEX_DRIVER, NULL);
540 mutex_init(&bd->d_statemutex, NULL, MUTEX_DRIVER, NULL);
541 cv_init(&bd->d_statecv, NULL, CV_DRIVER, NULL);
542
543 list_create(&bd->d_waitq, sizeof (bd_xfer_impl_t),
544 offsetof(struct bd_xfer_impl, i_linkage));
545 list_create(&bd->d_runq, sizeof (bd_xfer_impl_t),
546 offsetof(struct bd_xfer_impl, i_linkage));
547
548 bd->d_cache = kmem_cache_create(kcache, sizeof (bd_xfer_impl_t), 8,
549 bd_xfer_ctor, bd_xfer_dtor, NULL, bd, NULL, 0);
550
551 bd->d_ksp = kstat_create(ddi_driver_name(dip), inst, NULL, "disk",
552 KSTAT_TYPE_IO, 1, KSTAT_FLAG_PERSISTENT);
553 if (bd->d_ksp != NULL) {
554 bd->d_ksp->ks_lock = &bd->d_iomutex;
555 kstat_install(bd->d_ksp);
556 bd->d_kiop = bd->d_ksp->ks_data;
557 } else {
558 /*
559 * Even if we cannot create the kstat, we create a
560 * scratch kstat. The reason for this is to ensure
561 * that we can update the kstat all of the time,
562 * without adding an extra branch instruction.
563 */
564 bd->d_kiop = kmem_zalloc(sizeof (kstat_io_t), KM_SLEEP);
565 }
566
567 cmlb_alloc_handle(&bd->d_cmlbh);
568
569 bd->d_state = DKIO_NONE;
570
571 bzero(&drive, sizeof (drive));
572 bd->d_ops.o_drive_info(bd->d_private, &drive);
573 bd->d_qsize = drive.d_qsize;
574 bd->d_removable = drive.d_removable;
575 bd->d_hotpluggable = drive.d_hotpluggable;
576
577 if (drive.d_maxxfer && drive.d_maxxfer < bd->d_maxxfer)
578 bd->d_maxxfer = drive.d_maxxfer;
579
580 bd_create_inquiry_props(dip, &drive);
581
582 bd_create_errstats(bd, inst, &drive);
583 bd_init_errstats(bd, &drive);
584 bd_update_state(bd);
585
586 rv = cmlb_attach(dip, &bd_tg_ops, DTYPE_DIRECT,
587 bd->d_removable, bd->d_hotpluggable,
588 drive.d_lun >= 0 ? DDI_NT_BLOCK_CHAN : DDI_NT_BLOCK,
589 CMLB_FAKE_LABEL_ONE_PARTITION, bd->d_cmlbh, 0);
590 if (rv != 0) {
591 cmlb_free_handle(&bd->d_cmlbh);
592 kmem_cache_destroy(bd->d_cache);
593 mutex_destroy(&bd->d_iomutex);
594 mutex_destroy(&bd->d_ocmutex);
595 mutex_destroy(&bd->d_statemutex);
596 cv_destroy(&bd->d_statecv);
597 list_destroy(&bd->d_waitq);
598 list_destroy(&bd->d_runq);
599 if (bd->d_ksp != NULL) {
600 kstat_delete(bd->d_ksp);
601 bd->d_ksp = NULL;
602 } else {
603 kmem_free(bd->d_kiop, sizeof (kstat_io_t));
604 }
605 ddi_soft_state_free(bd_state, inst);
606 return (DDI_FAILURE);
607 }
608
609 if (bd->d_ops.o_devid_init != NULL) {
610 rv = bd->d_ops.o_devid_init(bd->d_private, dip, &bd->d_devid);
611 if (rv == DDI_SUCCESS) {
612 if (ddi_devid_register(dip, bd->d_devid) !=
613 DDI_SUCCESS) {
614 cmn_err(CE_WARN,
615 "%s: unable to register devid", name);
616 }
617 }
618 }
619
620 /*
621 * Add a zero-length attribute to tell the world we support
622 * kernel ioctls (for layered drivers). Also set up properties
623 * used by HAL to identify removable media.
624 */
625 (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
626 DDI_KERNEL_IOCTL, NULL, 0);
627 if (bd->d_removable) {
628 (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
629 "removable-media", NULL, 0);
630 }
631 if (bd->d_hotpluggable) {
632 (void) ddi_prop_create(DDI_DEV_T_NONE, dip, DDI_PROP_CANSLEEP,
633 "hotpluggable", NULL, 0);
634 }
635
636 ddi_report_dev(dip);
637
638 return (DDI_SUCCESS);
639 }
640
641 static int
642 bd_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
643 {
644 bd_t *bd;
645
646 bd = ddi_get_driver_private(dip);
647
648 switch (cmd) {
649 case DDI_DETACH:
650 break;
651 case DDI_SUSPEND:
652 /* We don't suspend, but our parent does */
653 return (DDI_SUCCESS);
654 default:
655 return (DDI_FAILURE);
656 }
657 if (bd->d_ksp != NULL) {
658 kstat_delete(bd->d_ksp);
659 bd->d_ksp = NULL;
660 } else {
661 kmem_free(bd->d_kiop, sizeof (kstat_io_t));
662 }
663
664 if (bd->d_errstats != NULL) {
665 kstat_delete(bd->d_errstats);
666 bd->d_errstats = NULL;
667 } else {
668 kmem_free(bd->d_kerr, sizeof (struct bd_errstats));
669 mutex_destroy(bd->d_errmutex);
670 }
671
672 cmlb_detach(bd->d_cmlbh, 0);
673 cmlb_free_handle(&bd->d_cmlbh);
674 if (bd->d_devid)
675 ddi_devid_free(bd->d_devid);
676 kmem_cache_destroy(bd->d_cache);
677 mutex_destroy(&bd->d_iomutex);
678 mutex_destroy(&bd->d_ocmutex);
679 mutex_destroy(&bd->d_statemutex);
680 cv_destroy(&bd->d_statecv);
681 list_destroy(&bd->d_waitq);
682 list_destroy(&bd->d_runq);
683 ddi_soft_state_free(bd_state, ddi_get_instance(dip));
684 return (DDI_SUCCESS);
685 }
686
687 static int
688 bd_xfer_ctor(void *buf, void *arg, int kmflag)
689 {
690 bd_xfer_impl_t *xi;
691 bd_t *bd = arg;
692 int (*dcb)(caddr_t);
693
694 if (kmflag == KM_PUSHPAGE || kmflag == KM_SLEEP) {
695 dcb = DDI_DMA_SLEEP;
696 } else {
697 dcb = DDI_DMA_DONTWAIT;
698 }
699
700 xi = buf;
701 bzero(xi, sizeof (*xi));
702 xi->i_bd = bd;
703
704 if (bd->d_use_dma) {
705 if (ddi_dma_alloc_handle(bd->d_dip, &bd->d_dma, dcb, NULL,
706 &xi->i_dmah) != DDI_SUCCESS) {
707 return (-1);
708 }
709 }
710
711 return (0);
712 }
713
714 static void
715 bd_xfer_dtor(void *buf, void *arg)
716 {
717 bd_xfer_impl_t *xi = buf;
718
719 _NOTE(ARGUNUSED(arg));
720
721 if (xi->i_dmah)
722 ddi_dma_free_handle(&xi->i_dmah);
723 xi->i_dmah = NULL;
724 }
725
726 static bd_xfer_impl_t *
727 bd_xfer_alloc(bd_t *bd, struct buf *bp, int (*func)(void *, bd_xfer_t *),
728 int kmflag)
729 {
730 bd_xfer_impl_t *xi;
731 int rv = 0;
732 int status;
733 unsigned dir;
734 int (*cb)(caddr_t);
735 size_t len;
736 uint32_t shift;
737
738 if (kmflag == KM_SLEEP) {
739 cb = DDI_DMA_SLEEP;
740 } else {
741 cb = DDI_DMA_DONTWAIT;
742 }
743
744 xi = kmem_cache_alloc(bd->d_cache, kmflag);
745 if (xi == NULL) {
746 bioerror(bp, ENOMEM);
747 return (NULL);
748 }
749
750 ASSERT(bp);
751
752 xi->i_bp = bp;
753 xi->i_func = func;
754 xi->i_blkno = bp->b_lblkno;
755
756 if (bp->b_bcount == 0) {
757 xi->i_len = 0;
758 xi->i_nblks = 0;
759 xi->i_kaddr = NULL;
760 xi->i_resid = 0;
761 xi->i_num_win = 0;
762 goto done;
763 }
764
765 if (bp->b_flags & B_READ) {
766 dir = DDI_DMA_READ;
767 xi->i_func = bd->d_ops.o_read;
768 } else {
769 dir = DDI_DMA_WRITE;
770 xi->i_func = bd->d_ops.o_write;
771 }
772
773 shift = bd->d_blkshift;
774 xi->i_blkshift = shift;
775
776 if (!bd->d_use_dma) {
777 bp_mapin(bp);
778 rv = 0;
779 xi->i_offset = 0;
780 xi->i_num_win =
781 (bp->b_bcount + (bd->d_maxxfer - 1)) / bd->d_maxxfer;
782 xi->i_cur_win = 0;
783 xi->i_len = min(bp->b_bcount, bd->d_maxxfer);
784 xi->i_nblks = xi->i_len >> shift;
785 xi->i_kaddr = bp->b_un.b_addr;
786 xi->i_resid = bp->b_bcount;
787 } else {
788
789 /*
790 * We have to use consistent DMA if the address is misaligned.
791 */
792 if (((bp->b_flags & (B_PAGEIO | B_REMAPPED)) != B_PAGEIO) &&
793 ((uintptr_t)bp->b_un.b_addr & 0x7)) {
794 dir |= DDI_DMA_CONSISTENT | DDI_DMA_PARTIAL;
795 } else {
796 dir |= DDI_DMA_STREAMING | DDI_DMA_PARTIAL;
797 }
798
799 status = ddi_dma_buf_bind_handle(xi->i_dmah, bp, dir, cb,
800 NULL, &xi->i_dmac, &xi->i_ndmac);
801 switch (status) {
802 case DDI_DMA_MAPPED:
803 xi->i_num_win = 1;
804 xi->i_cur_win = 0;
805 xi->i_offset = 0;
806 xi->i_len = bp->b_bcount;
807 xi->i_nblks = xi->i_len >> shift;
808 xi->i_resid = bp->b_bcount;
809 rv = 0;
810 break;
811 case DDI_DMA_PARTIAL_MAP:
812 xi->i_cur_win = 0;
813
814 if ((ddi_dma_numwin(xi->i_dmah, &xi->i_num_win) !=
815 DDI_SUCCESS) ||
816 (ddi_dma_getwin(xi->i_dmah, 0, &xi->i_offset,
817 &len, &xi->i_dmac, &xi->i_ndmac) !=
818 DDI_SUCCESS) ||
819 (P2PHASE(len, shift) != 0)) {
820 (void) ddi_dma_unbind_handle(xi->i_dmah);
821 rv = EFAULT;
822 goto done;
823 }
824 xi->i_len = len;
825 xi->i_nblks = xi->i_len >> shift;
826 xi->i_resid = bp->b_bcount;
827 rv = 0;
828 break;
829 case DDI_DMA_NORESOURCES:
830 rv = EAGAIN;
831 goto done;
832 case DDI_DMA_TOOBIG:
833 rv = EINVAL;
834 goto done;
835 case DDI_DMA_NOMAPPING:
836 case DDI_DMA_INUSE:
837 default:
838 rv = EFAULT;
839 goto done;
840 }
841 }
842
843 done:
844 if (rv != 0) {
845 kmem_cache_free(bd->d_cache, xi);
846 bioerror(bp, rv);
847 return (NULL);
848 }
849
850 return (xi);
851 }
852
853 static void
854 bd_xfer_free(bd_xfer_impl_t *xi)
855 {
856 if (xi->i_dmah) {
857 (void) ddi_dma_unbind_handle(xi->i_dmah);
858 }
859 kmem_cache_free(xi->i_bd->d_cache, xi);
860 }
861
862 static int
863 bd_open(dev_t *devp, int flag, int otyp, cred_t *credp)
864 {
865 dev_t dev = *devp;
866 bd_t *bd;
867 minor_t part;
868 minor_t inst;
869 uint64_t mask;
870 boolean_t ndelay;
871 int rv;
872 diskaddr_t nblks;
873 diskaddr_t lba;
874
875 _NOTE(ARGUNUSED(credp));
876
877 part = BDPART(dev);
878 inst = BDINST(dev);
879
880 if (otyp >= OTYPCNT)
881 return (EINVAL);
882
883 ndelay = (flag & (FNDELAY | FNONBLOCK)) ? B_TRUE : B_FALSE;
884
885 /*
886 * Block any DR events from changing the set of registered
887 * devices while we function.
888 */
889 rw_enter(&bd_lock, RW_READER);
890 if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {
891 rw_exit(&bd_lock);
892 return (ENXIO);
893 }
894
895 mutex_enter(&bd->d_ocmutex);
896
897 ASSERT(part < 64);
898 mask = (1U << part);
899
900 bd_update_state(bd);
901
902 if (cmlb_validate(bd->d_cmlbh, 0, 0) != 0) {
903
904 /* non-blocking opens are allowed to succeed */
905 if (!ndelay) {
906 rv = ENXIO;
907 goto done;
908 }
909 } else if (cmlb_partinfo(bd->d_cmlbh, part, &nblks, &lba,
910 NULL, NULL, 0) == 0) {
911
912 /*
913 * We read the partinfo, verify valid ranges. If the
914 * partition is invalid, and we aren't blocking or
915 * doing a raw access, then fail. (Non-blocking and
916 * raw accesses can still succeed to allow a disk with
917 * bad partition data to opened by format and fdisk.)
918 */
919 if ((!nblks) && ((!ndelay) || (otyp != OTYP_CHR))) {
920 rv = ENXIO;
921 goto done;
922 }
923 } else if (!ndelay) {
924 /*
925 * cmlb_partinfo failed -- invalid partition or no
926 * disk label.
927 */
928 rv = ENXIO;
929 goto done;
930 }
931
932 if ((flag & FWRITE) && bd->d_rdonly) {
933 rv = EROFS;
934 goto done;
935 }
936
937 if ((bd->d_open_excl) & (mask)) {
938 rv = EBUSY;
939 goto done;
940 }
941 if (flag & FEXCL) {
942 if (bd->d_open_lyr[part]) {
943 rv = EBUSY;
944 goto done;
945 }
946 for (int i = 0; i < OTYP_LYR; i++) {
947 if (bd->d_open_reg[i] & mask) {
948 rv = EBUSY;
949 goto done;
950 }
951 }
952 }
953
954 if (otyp == OTYP_LYR) {
955 bd->d_open_lyr[part]++;
956 } else {
957 bd->d_open_reg[otyp] |= mask;
958 }
959 if (flag & FEXCL) {
960 bd->d_open_excl |= mask;
961 }
962
963 rv = 0;
964 done:
965 mutex_exit(&bd->d_ocmutex);
966 rw_exit(&bd_lock);
967
968 return (rv);
969 }
970
971 static int
972 bd_close(dev_t dev, int flag, int otyp, cred_t *credp)
973 {
974 bd_t *bd;
975 minor_t inst;
976 minor_t part;
977 uint64_t mask;
978 boolean_t last = B_TRUE;
979
980 _NOTE(ARGUNUSED(flag));
981 _NOTE(ARGUNUSED(credp));
982
983 part = BDPART(dev);
984 inst = BDINST(dev);
985
986 ASSERT(part < 64);
987 mask = (1U << part);
988
989 rw_enter(&bd_lock, RW_READER);
990
991 if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {
992 rw_exit(&bd_lock);
993 return (ENXIO);
994 }
995
996 mutex_enter(&bd->d_ocmutex);
997 if (bd->d_open_excl & mask) {
998 bd->d_open_excl &= ~mask;
999 }
1000 if (otyp == OTYP_LYR) {
1001 bd->d_open_lyr[part]--;
1002 } else {
1003 bd->d_open_reg[otyp] &= ~mask;
1004 }
1005 for (int i = 0; i < 64; i++) {
1006 if (bd->d_open_lyr[part]) {
1007 last = B_FALSE;
1008 }
1009 }
1010 for (int i = 0; last && (i < OTYP_LYR); i++) {
1011 if (bd->d_open_reg[i]) {
1012 last = B_FALSE;
1013 }
1014 }
1015 mutex_exit(&bd->d_ocmutex);
1016
1017 if (last) {
1018 cmlb_invalidate(bd->d_cmlbh, 0);
1019 }
1020 rw_exit(&bd_lock);
1021
1022 return (0);
1023 }
1024
1025 static int
1026 bd_dump(dev_t dev, caddr_t caddr, daddr_t blkno, int nblk)
1027 {
1028 minor_t inst;
1029 minor_t part;
1030 diskaddr_t pstart;
1031 diskaddr_t psize;
1032 bd_t *bd;
1033 bd_xfer_impl_t *xi;
1034 buf_t *bp;
1035 int rv;
1036
1037 rw_enter(&bd_lock, RW_READER);
1038
1039 part = BDPART(dev);
1040 inst = BDINST(dev);
1041
1042 if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {
1043 rw_exit(&bd_lock);
1044 return (ENXIO);
1045 }
1046 /*
1047 * do cmlb, but do it synchronously unless we already have the
1048 * partition (which we probably should.)
1049 */
1050 if (cmlb_partinfo(bd->d_cmlbh, part, &psize, &pstart, NULL, NULL,
1051 (void *)1)) {
1052 rw_exit(&bd_lock);
1053 return (ENXIO);
1054 }
1055
1056 if ((blkno + nblk) > psize) {
1057 rw_exit(&bd_lock);
1058 return (EINVAL);
1059 }
1060 bp = getrbuf(KM_NOSLEEP);
1061 if (bp == NULL) {
1062 rw_exit(&bd_lock);
1063 return (ENOMEM);
1064 }
1065
1066 bp->b_bcount = nblk << bd->d_blkshift;
1067 bp->b_resid = bp->b_bcount;
1068 bp->b_lblkno = blkno;
1069 bp->b_un.b_addr = caddr;
1070
1071 xi = bd_xfer_alloc(bd, bp, bd->d_ops.o_write, KM_NOSLEEP);
1072 if (xi == NULL) {
1073 rw_exit(&bd_lock);
1074 freerbuf(bp);
1075 return (ENOMEM);
1076 }
1077 xi->i_blkno = blkno + pstart;
1078 xi->i_flags = BD_XFER_POLL;
1079 bd_submit(bd, xi);
1080 rw_exit(&bd_lock);
1081
1082 /*
1083 * Generally, we should have run this entirely synchronously
1084 * at this point and the biowait call should be a no-op. If
1085 * it didn't happen this way, it's a bug in the underlying
1086 * driver not honoring BD_XFER_POLL.
1087 */
1088 (void) biowait(bp);
1089 rv = geterror(bp);
1090 freerbuf(bp);
1091 return (rv);
1092 }
1093
1094 void
1095 bd_minphys(struct buf *bp)
1096 {
1097 minor_t inst;
1098 bd_t *bd;
1099 inst = BDINST(bp->b_edev);
1100
1101 bd = ddi_get_soft_state(bd_state, inst);
1102
1103 /*
1104 * In a non-debug kernel, bd_strategy will catch !bd as
1105 * well, and will fail nicely.
1106 */
1107 ASSERT(bd);
1108
1109 if (bp->b_bcount > bd->d_maxxfer)
1110 bp->b_bcount = bd->d_maxxfer;
1111 }
1112
1113 static int
1114 bd_read(dev_t dev, struct uio *uio, cred_t *credp)
1115 {
1116 _NOTE(ARGUNUSED(credp));
1117 return (physio(bd_strategy, NULL, dev, B_READ, bd_minphys, uio));
1118 }
1119
1120 static int
1121 bd_write(dev_t dev, struct uio *uio, cred_t *credp)
1122 {
1123 _NOTE(ARGUNUSED(credp));
1124 return (physio(bd_strategy, NULL, dev, B_WRITE, bd_minphys, uio));
1125 }
1126
1127 static int
1128 bd_aread(dev_t dev, struct aio_req *aio, cred_t *credp)
1129 {
1130 _NOTE(ARGUNUSED(credp));
1131 return (aphysio(bd_strategy, anocancel, dev, B_READ, bd_minphys, aio));
1132 }
1133
1134 static int
1135 bd_awrite(dev_t dev, struct aio_req *aio, cred_t *credp)
1136 {
1137 _NOTE(ARGUNUSED(credp));
1138 return (aphysio(bd_strategy, anocancel, dev, B_WRITE, bd_minphys, aio));
1139 }
1140
1141 static int
1142 bd_strategy(struct buf *bp)
1143 {
1144 minor_t inst;
1145 minor_t part;
1146 bd_t *bd;
1147 diskaddr_t p_lba;
1148 diskaddr_t p_nblks;
1149 diskaddr_t b_nblks;
1150 bd_xfer_impl_t *xi;
1151 uint32_t shift;
1152 int (*func)(void *, bd_xfer_t *);
1153
1154 part = BDPART(bp->b_edev);
1155 inst = BDINST(bp->b_edev);
1156
1157 ASSERT(bp);
1158
1159 bp->b_resid = bp->b_bcount;
1160
1161 if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {
1162 bioerror(bp, ENXIO);
1163 biodone(bp);
1164 return (0);
1165 }
1166
1167 if (cmlb_partinfo(bd->d_cmlbh, part, &p_nblks, &p_lba,
1168 NULL, NULL, 0)) {
1169 bioerror(bp, ENXIO);
1170 biodone(bp);
1171 return (0);
1172 }
1173
1174 shift = bd->d_blkshift;
1175
1176 if ((P2PHASE(bp->b_bcount, (1U << shift)) != 0) ||
1177 (bp->b_lblkno > p_nblks)) {
1178 bioerror(bp, ENXIO);
1179 biodone(bp);
1180 return (0);
1181 }
1182 b_nblks = bp->b_bcount >> shift;
1183 if ((bp->b_lblkno == p_nblks) || (bp->b_bcount == 0)) {
1184 biodone(bp);
1185 return (0);
1186 }
1187
1188 if ((b_nblks + bp->b_lblkno) > p_nblks) {
1189 bp->b_resid = ((bp->b_lblkno + b_nblks - p_nblks) << shift);
1190 bp->b_bcount -= bp->b_resid;
1191 } else {
1192 bp->b_resid = 0;
1193 }
1194 func = (bp->b_flags & B_READ) ? bd->d_ops.o_read : bd->d_ops.o_write;
1195
1196 xi = bd_xfer_alloc(bd, bp, func, KM_NOSLEEP);
1197 if (xi == NULL) {
1198 xi = bd_xfer_alloc(bd, bp, func, KM_PUSHPAGE);
1199 }
1200 if (xi == NULL) {
1201 /* bd_request_alloc will have done bioerror */
1202 biodone(bp);
1203 return (0);
1204 }
1205 xi->i_blkno = bp->b_lblkno + p_lba;
1206
1207 bd_submit(bd, xi);
1208
1209 return (0);
1210 }
1211
1212 static int
1213 bd_ioctl(dev_t dev, int cmd, intptr_t arg, int flag, cred_t *credp, int *rvalp)
1214 {
1215 minor_t inst;
1216 uint16_t part;
1217 bd_t *bd;
1218 void *ptr = (void *)arg;
1219 int rv;
1220
1221 part = BDPART(dev);
1222 inst = BDINST(dev);
1223
1224 if ((bd = ddi_get_soft_state(bd_state, inst)) == NULL) {
1225 return (ENXIO);
1226 }
1227
1228 rv = cmlb_ioctl(bd->d_cmlbh, dev, cmd, arg, flag, credp, rvalp, 0);
1229 if (rv != ENOTTY)
1230 return (rv);
1231
1232 if (rvalp != NULL) {
1233 /* the return value of the ioctl is 0 by default */
1234 *rvalp = 0;
1235 }
1236
1237 switch (cmd) {
1238 case DKIOCGMEDIAINFO: {
1239 struct dk_minfo minfo;
1240
1241 /* make sure our state information is current */
1242 bd_update_state(bd);
1243 bzero(&minfo, sizeof (minfo));
1244 minfo.dki_media_type = DK_FIXED_DISK;
1245 minfo.dki_lbsize = (1U << bd->d_blkshift);
1246 minfo.dki_capacity = bd->d_numblks;
1247 if (ddi_copyout(&minfo, ptr, sizeof (minfo), flag)) {
1248 return (EFAULT);
1249 }
1250 return (0);
1251 }
1252 case DKIOCGMEDIAINFOEXT: {
1253 struct dk_minfo_ext miext;
1254
1255 /* make sure our state information is current */
1256 bd_update_state(bd);
1257 bzero(&miext, sizeof (miext));
1258 miext.dki_media_type = DK_FIXED_DISK;
1259 miext.dki_lbsize = (1U << bd->d_blkshift);
1260 miext.dki_pbsize = (1U << bd->d_pblkshift);
1261 miext.dki_capacity = bd->d_numblks;
1262 if (ddi_copyout(&miext, ptr, sizeof (miext), flag)) {
1263 return (EFAULT);
1264 }
1265 return (0);
1266 }
1267 case DKIOCINFO: {
1268 struct dk_cinfo cinfo;
1269 bzero(&cinfo, sizeof (cinfo));
1270 cinfo.dki_ctype = DKC_BLKDEV;
1271 cinfo.dki_cnum = ddi_get_instance(ddi_get_parent(bd->d_dip));
1272 (void) snprintf(cinfo.dki_cname, sizeof (cinfo.dki_cname),
1273 "%s", ddi_driver_name(ddi_get_parent(bd->d_dip)));
1274 (void) snprintf(cinfo.dki_dname, sizeof (cinfo.dki_dname),
1275 "%s", ddi_driver_name(bd->d_dip));
1276 cinfo.dki_unit = inst;
1277 cinfo.dki_flags = DKI_FMTVOL;
1278 cinfo.dki_partition = part;
1279 cinfo.dki_maxtransfer = bd->d_maxxfer / DEV_BSIZE;
1280 cinfo.dki_addr = 0;
1281 cinfo.dki_slave = 0;
1282 cinfo.dki_space = 0;
1283 cinfo.dki_prio = 0;
1284 cinfo.dki_vec = 0;
1285 if (ddi_copyout(&cinfo, ptr, sizeof (cinfo), flag)) {
1286 return (EFAULT);
1287 }
1288 return (0);
1289 }
1290 case DKIOCREMOVABLE: {
1291 int i;
1292 i = bd->d_removable ? 1 : 0;
1293 if (ddi_copyout(&i, ptr, sizeof (i), flag)) {
1294 return (EFAULT);
1295 }
1296 return (0);
1297 }
1298 case DKIOCHOTPLUGGABLE: {
1299 int i;
1300 i = bd->d_hotpluggable ? 1 : 0;
1301 if (ddi_copyout(&i, ptr, sizeof (i), flag)) {
1302 return (EFAULT);
1303 }
1304 return (0);
1305 }
1306 case DKIOCREADONLY: {
1307 int i;
1308 i = bd->d_rdonly ? 1 : 0;
1309 if (ddi_copyout(&i, ptr, sizeof (i), flag)) {
1310 return (EFAULT);
1311 }
1312 return (0);
1313 }
1314 case DKIOCSOLIDSTATE: {
1315 int i;
1316 i = bd->d_ssd ? 1 : 0;
1317 if (ddi_copyout(&i, ptr, sizeof (i), flag)) {
1318 return (EFAULT);
1319 }
1320 return (0);
1321 }
1322 case DKIOCSTATE: {
1323 enum dkio_state state;
1324 if (ddi_copyin(ptr, &state, sizeof (state), flag)) {
1325 return (EFAULT);
1326 }
1327 if ((rv = bd_check_state(bd, &state)) != 0) {
1328 return (rv);
1329 }
1330 if (ddi_copyout(&state, ptr, sizeof (state), flag)) {
1331 return (EFAULT);
1332 }
1333 return (0);
1334 }
1335 case DKIOCFLUSHWRITECACHE: {
1336 struct dk_callback *dkc = NULL;
1337
1338 if (flag & FKIOCTL)
1339 dkc = (void *)arg;
1340
1341 rv = bd_flush_write_cache(bd, dkc);
1342 return (rv);
1343 }
1344
1345 default:
1346 break;
1347
1348 }
1349 return (ENOTTY);
1350 }
1351
1352 static int
1353 bd_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int mod_flags,
1354 char *name, caddr_t valuep, int *lengthp)
1355 {
1356 bd_t *bd;
1357
1358 bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip));
1359 if (bd == NULL)
1360 return (ddi_prop_op(dev, dip, prop_op, mod_flags,
1361 name, valuep, lengthp));
1362
1363 return (cmlb_prop_op(bd->d_cmlbh, dev, dip, prop_op, mod_flags, name,
1364 valuep, lengthp, BDPART(dev), 0));
1365 }
1366
1367
1368 static int
1369 bd_tg_rdwr(dev_info_t *dip, uchar_t cmd, void *bufaddr, diskaddr_t start,
1370 size_t length, void *tg_cookie)
1371 {
1372 bd_t *bd;
1373 buf_t *bp;
1374 bd_xfer_impl_t *xi;
1375 int rv;
1376 int (*func)(void *, bd_xfer_t *);
1377 int kmflag;
1378
1379 /*
1380 * If we are running in polled mode (such as during dump(9e)
1381 * execution), then we cannot sleep for kernel allocations.
1382 */
1383 kmflag = tg_cookie ? KM_NOSLEEP : KM_SLEEP;
1384
1385 bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip));
1386
1387 if (P2PHASE(length, (1U << bd->d_blkshift)) != 0) {
1388 /* We can only transfer whole blocks at a time! */
1389 return (EINVAL);
1390 }
1391
1392 if ((bp = getrbuf(kmflag)) == NULL) {
1393 return (ENOMEM);
1394 }
1395
1396 switch (cmd) {
1397 case TG_READ:
1398 bp->b_flags = B_READ;
1399 func = bd->d_ops.o_read;
1400 break;
1401 case TG_WRITE:
1402 bp->b_flags = B_WRITE;
1403 func = bd->d_ops.o_write;
1404 break;
1405 default:
1406 freerbuf(bp);
1407 return (EINVAL);
1408 }
1409
1410 bp->b_un.b_addr = bufaddr;
1411 bp->b_bcount = length;
1412 xi = bd_xfer_alloc(bd, bp, func, kmflag);
1413 if (xi == NULL) {
1414 rv = geterror(bp);
1415 freerbuf(bp);
1416 return (rv);
1417 }
1418 xi->i_flags = tg_cookie ? BD_XFER_POLL : 0;
1419 xi->i_blkno = start;
1420 bd_submit(bd, xi);
1421 (void) biowait(bp);
1422 rv = geterror(bp);
1423 freerbuf(bp);
1424
1425 return (rv);
1426 }
1427
1428 static int
1429 bd_tg_getinfo(dev_info_t *dip, int cmd, void *arg, void *tg_cookie)
1430 {
1431 bd_t *bd;
1432
1433 _NOTE(ARGUNUSED(tg_cookie));
1434 bd = ddi_get_soft_state(bd_state, ddi_get_instance(dip));
1435
1436 switch (cmd) {
1437 case TG_GETPHYGEOM:
1438 case TG_GETVIRTGEOM:
1439 /*
1440 * We don't have any "geometry" as such, let cmlb
1441 * fabricate something.
1442 */
1443 return (ENOTTY);
1444
1445 case TG_GETCAPACITY:
1446 bd_update_state(bd);
1447 *(diskaddr_t *)arg = bd->d_numblks;
1448 return (0);
1449
1450 case TG_GETBLOCKSIZE:
1451 *(uint32_t *)arg = (1U << bd->d_blkshift);
1452 return (0);
1453
1454 case TG_GETATTR:
1455 /*
1456 * It turns out that cmlb really doesn't do much for
1457 * non-writable media, but lets make the information
1458 * available for it in case it does more in the
1459 * future. (The value is currently used for
1460 * triggering special behavior for CD-ROMs.)
1461 */
1462 bd_update_state(bd);
1463 ((tg_attribute_t *)arg)->media_is_writable =
1464 bd->d_rdonly ? B_FALSE : B_TRUE;
1465 ((tg_attribute_t *)arg)->media_is_solid_state = bd->d_ssd;
1466 return (0);
1467
1468 default:
1469 return (EINVAL);
1470 }
1471 }
1472
1473
1474 static void
1475 bd_sched(bd_t *bd)
1476 {
1477 bd_xfer_impl_t *xi;
1478 struct buf *bp;
1479 int rv;
1480
1481 mutex_enter(&bd->d_iomutex);
1482
1483 while ((bd->d_qactive < bd->d_qsize) &&
1484 ((xi = list_remove_head(&bd->d_waitq)) != NULL)) {
1485 bd->d_qactive++;
1486 kstat_waitq_to_runq(bd->d_kiop);
1487 list_insert_tail(&bd->d_runq, xi);
1488
1489 /*
1490 * Submit the job to the driver. We drop the I/O mutex
1491 * so that we can deal with the case where the driver
1492 * completion routine calls back into us synchronously.
1493 */
1494
1495 mutex_exit(&bd->d_iomutex);
1496
1497 rv = xi->i_func(bd->d_private, &xi->i_public);
1498 if (rv != 0) {
1499 bp = xi->i_bp;
1500 bioerror(bp, rv);
1501 biodone(bp);
1502
1503 atomic_inc_32(&bd->d_kerr->bd_transerrs.value.ui32);
1504
1505 mutex_enter(&bd->d_iomutex);
1506 bd->d_qactive--;
1507 kstat_runq_exit(bd->d_kiop);
1508 list_remove(&bd->d_runq, xi);
1509 bd_xfer_free(xi);
1510 } else {
1511 mutex_enter(&bd->d_iomutex);
1512 }
1513 }
1514
1515 mutex_exit(&bd->d_iomutex);
1516 }
1517
1518 static void
1519 bd_submit(bd_t *bd, bd_xfer_impl_t *xi)
1520 {
1521 mutex_enter(&bd->d_iomutex);
1522 list_insert_tail(&bd->d_waitq, xi);
1523 kstat_waitq_enter(bd->d_kiop);
1524 mutex_exit(&bd->d_iomutex);
1525
1526 bd_sched(bd);
1527 }
1528
1529 static void
1530 bd_runq_exit(bd_xfer_impl_t *xi, int err)
1531 {
1532 bd_t *bd = xi->i_bd;
1533 buf_t *bp = xi->i_bp;
1534
1535 mutex_enter(&bd->d_iomutex);
1536 bd->d_qactive--;
1537 kstat_runq_exit(bd->d_kiop);
1538 list_remove(&bd->d_runq, xi);
1539 mutex_exit(&bd->d_iomutex);
1540
1541 if (err == 0) {
1542 if (bp->b_flags & B_READ) {
1543 bd->d_kiop->reads++;
1544 bd->d_kiop->nread += (bp->b_bcount - xi->i_resid);
1545 } else {
1546 bd->d_kiop->writes++;
1547 bd->d_kiop->nwritten += (bp->b_bcount - xi->i_resid);
1548 }
1549 }
1550 bd_sched(bd);
1551 }
1552
1553 static void
1554 bd_update_state(bd_t *bd)
1555 {
1556 enum dkio_state state = DKIO_INSERTED;
1557 boolean_t docmlb = B_FALSE;
1558 bd_media_t media;
1559
1560 bzero(&media, sizeof (media));
1561
1562 mutex_enter(&bd->d_statemutex);
1563 if (bd->d_ops.o_media_info(bd->d_private, &media) != 0) {
1564 bd->d_numblks = 0;
1565 state = DKIO_EJECTED;
1566 goto done;
1567 }
1568
1569 if ((media.m_blksize < 512) ||
1570 (!ISP2(media.m_blksize)) ||
1571 (P2PHASE(bd->d_maxxfer, media.m_blksize))) {
1572 cmn_err(CE_WARN, "%s%d: Invalid media block size (%d)",
1573 ddi_driver_name(bd->d_dip), ddi_get_instance(bd->d_dip),
1574 media.m_blksize);
1575 /*
1576 * We can't use the media, treat it as not present.
1577 */
1578 state = DKIO_EJECTED;
1579 bd->d_numblks = 0;
1580 goto done;
1581 }
1582
1583 if (((1U << bd->d_blkshift) != media.m_blksize) ||
1584 (bd->d_numblks != media.m_nblks)) {
1585 /* Device size changed */
1586 docmlb = B_TRUE;
1587 }
1588
1589 bd->d_blkshift = ddi_ffs(media.m_blksize) - 1;
1590 bd->d_pblkshift = bd->d_blkshift;
1591 bd->d_numblks = media.m_nblks;
1592 bd->d_rdonly = media.m_readonly;
1593 bd->d_ssd = media.m_solidstate;
1594
1595 /*
1596 * Only use the supplied physical block size if it is non-zero,
1597 * greater or equal to the block size, and a power of 2. Ignore it
1598 * if not, it's just informational and we can still use the media.
1599 */
1600 if ((media.m_pblksize != 0) &&
1601 (media.m_pblksize >= media.m_blksize) &&
1602 (ISP2(media.m_pblksize)))
1603 bd->d_pblkshift = ddi_ffs(media.m_pblksize) - 1;
1604
1605 done:
1606 if (state != bd->d_state) {
1607 bd->d_state = state;
1608 cv_broadcast(&bd->d_statecv);
1609 docmlb = B_TRUE;
1610 }
1611 mutex_exit(&bd->d_statemutex);
1612
1613 bd->d_kerr->bd_capacity.value.ui64 = bd->d_numblks << bd->d_blkshift;
1614
1615 if (docmlb) {
1616 if (state == DKIO_INSERTED) {
1617 (void) cmlb_validate(bd->d_cmlbh, 0, 0);
1618 } else {
1619 cmlb_invalidate(bd->d_cmlbh, 0);
1620 }
1621 }
1622 }
1623
1624 static int
1625 bd_check_state(bd_t *bd, enum dkio_state *state)
1626 {
1627 clock_t when;
1628
1629 for (;;) {
1630
1631 bd_update_state(bd);
1632
1633 mutex_enter(&bd->d_statemutex);
1634
1635 if (bd->d_state != *state) {
1636 *state = bd->d_state;
1637 mutex_exit(&bd->d_statemutex);
1638 break;
1639 }
1640
1641 when = drv_usectohz(1000000);
1642 if (cv_reltimedwait_sig(&bd->d_statecv, &bd->d_statemutex,
1643 when, TR_CLOCK_TICK) == 0) {
1644 mutex_exit(&bd->d_statemutex);
1645 return (EINTR);
1646 }
1647
1648 mutex_exit(&bd->d_statemutex);
1649 }
1650
1651 return (0);
1652 }
1653
1654 static int
1655 bd_flush_write_cache_done(struct buf *bp)
1656 {
1657 struct dk_callback *dc = (void *)bp->b_private;
1658
1659 (*dc->dkc_callback)(dc->dkc_cookie, geterror(bp));
1660 kmem_free(dc, sizeof (*dc));
1661 freerbuf(bp);
1662 return (0);
1663 }
1664
1665 static int
1666 bd_flush_write_cache(bd_t *bd, struct dk_callback *dkc)
1667 {
1668 buf_t *bp;
1669 struct dk_callback *dc;
1670 bd_xfer_impl_t *xi;
1671 int rv;
1672
1673 if (bd->d_ops.o_sync_cache == NULL) {
1674 return (ENOTSUP);
1675 }
1676 if ((bp = getrbuf(KM_SLEEP)) == NULL) {
1677 return (ENOMEM);
1678 }
1679 bp->b_resid = 0;
1680 bp->b_bcount = 0;
1681
1682 xi = bd_xfer_alloc(bd, bp, bd->d_ops.o_sync_cache, KM_SLEEP);
1683 if (xi == NULL) {
1684 rv = geterror(bp);
1685 freerbuf(bp);
1686 return (rv);
1687 }
1688
1689 /* Make an asynchronous flush, but only if there is a callback */
1690 if (dkc != NULL && dkc->dkc_callback != NULL) {
1691 /* Make a private copy of the callback structure */
1692 dc = kmem_alloc(sizeof (*dc), KM_SLEEP);
1693 *dc = *dkc;
1694 bp->b_private = dc;
1695 bp->b_iodone = bd_flush_write_cache_done;
1696
1697 bd_submit(bd, xi);
1698 return (0);
1699 }
1700
1701 /* In case there is no callback, perform a synchronous flush */
1702 bd_submit(bd, xi);
1703 (void) biowait(bp);
1704 rv = geterror(bp);
1705 freerbuf(bp);
1706
1707 return (rv);
1708 }
1709
1710 /*
1711 * Nexus support.
1712 */
1713 int
1714 bd_bus_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop,
1715 void *arg, void *result)
1716 {
1717 bd_handle_t hdl;
1718
1719 switch (ctlop) {
1720 case DDI_CTLOPS_REPORTDEV:
1721 cmn_err(CE_CONT, "?Block device: %s@%s, %s%d\n",
1722 ddi_node_name(rdip), ddi_get_name_addr(rdip),
1723 ddi_driver_name(rdip), ddi_get_instance(rdip));
1724 return (DDI_SUCCESS);
1725
1726 case DDI_CTLOPS_INITCHILD:
1727 hdl = ddi_get_parent_data((dev_info_t *)arg);
1728 if (hdl == NULL) {
1729 return (DDI_NOT_WELL_FORMED);
1730 }
1731 ddi_set_name_addr((dev_info_t *)arg, hdl->h_addr);
1732 return (DDI_SUCCESS);
1733
1734 case DDI_CTLOPS_UNINITCHILD:
1735 ddi_set_name_addr((dev_info_t *)arg, NULL);
1736 ndi_prop_remove_all((dev_info_t *)arg);
1737 return (DDI_SUCCESS);
1738
1739 default:
1740 return (ddi_ctlops(dip, rdip, ctlop, arg, result));
1741 }
1742 }
1743
1744 /*
1745 * Functions for device drivers.
1746 */
1747 bd_handle_t
1748 bd_alloc_handle(void *private, bd_ops_t *ops, ddi_dma_attr_t *dma, int kmflag)
1749 {
1750 bd_handle_t hdl;
1751
1752 hdl = kmem_zalloc(sizeof (*hdl), kmflag);
1753 if (hdl != NULL) {
1754 hdl->h_ops = *ops;
1755 hdl->h_dma = dma;
1756 hdl->h_private = private;
1757 }
1758
1759 return (hdl);
1760 }
1761
1762 void
1763 bd_free_handle(bd_handle_t hdl)
1764 {
1765 kmem_free(hdl, sizeof (*hdl));
1766 }
1767
1768 int
1769 bd_attach_handle(dev_info_t *dip, bd_handle_t hdl)
1770 {
1771 dev_info_t *child;
1772 bd_drive_t drive = { 0 };
1773
1774 /* if drivers don't override this, make it assume none */
1775 drive.d_lun = -1;
1776 hdl->h_ops.o_drive_info(hdl->h_private, &drive);
1777
1778 hdl->h_parent = dip;
1779 hdl->h_name = "blkdev";
1780
1781 if (drive.d_lun >= 0) {
1782 (void) snprintf(hdl->h_addr, sizeof (hdl->h_addr), "%X,%X",
1783 drive.d_target, drive.d_lun);
1784 } else {
1785 (void) snprintf(hdl->h_addr, sizeof (hdl->h_addr), "%X",
1786 drive.d_target);
1787 }
1788 if (ndi_devi_alloc(dip, hdl->h_name, (pnode_t)DEVI_SID_NODEID,
1789 &child) != NDI_SUCCESS) {
1790 cmn_err(CE_WARN, "%s%d: unable to allocate node %s@%s",
1791 ddi_driver_name(dip), ddi_get_instance(dip),
1792 "blkdev", hdl->h_addr);
1793 return (DDI_FAILURE);
1794 }
1795
1796 ddi_set_parent_data(child, hdl);
1797 hdl->h_child = child;
1798
1799 if (ndi_devi_online(child, 0) == NDI_FAILURE) {
1800 cmn_err(CE_WARN, "%s%d: failed bringing node %s@%s online",
1801 ddi_driver_name(dip), ddi_get_instance(dip),
1802 hdl->h_name, hdl->h_addr);
1803 (void) ndi_devi_free(child);
1804 return (DDI_FAILURE);
1805 }
1806
1807 return (DDI_SUCCESS);
1808 }
1809
1810 int
1811 bd_detach_handle(bd_handle_t hdl)
1812 {
1813 int circ;
1814 int rv;
1815 char *devnm;
1816
1817 if (hdl->h_child == NULL) {
1818 return (DDI_SUCCESS);
1819 }
1820 ndi_devi_enter(hdl->h_parent, &circ);
1821 if (i_ddi_node_state(hdl->h_child) < DS_INITIALIZED) {
1822 rv = ddi_remove_child(hdl->h_child, 0);
1823 } else {
1824 devnm = kmem_alloc(MAXNAMELEN + 1, KM_SLEEP);
1825 (void) ddi_deviname(hdl->h_child, devnm);
1826 (void) devfs_clean(hdl->h_parent, devnm + 1, DV_CLEAN_FORCE);
1827 rv = ndi_devi_unconfig_one(hdl->h_parent, devnm + 1, NULL,
1828 NDI_DEVI_REMOVE | NDI_UNCONFIG);
1829 kmem_free(devnm, MAXNAMELEN + 1);
1830 }
1831 if (rv == 0) {
1832 hdl->h_child = NULL;
1833 }
1834
1835 ndi_devi_exit(hdl->h_parent, circ);
1836 return (rv = NDI_SUCCESS ? DDI_SUCCESS : DDI_FAILURE);
1837 }
1838
1839 void
1840 bd_xfer_done(bd_xfer_t *xfer, int err)
1841 {
1842 bd_xfer_impl_t *xi = (void *)xfer;
1843 buf_t *bp = xi->i_bp;
1844 int rv = DDI_SUCCESS;
1845 bd_t *bd = xi->i_bd;
1846 size_t len;
1847
1848 if (err != 0) {
1849 bd_runq_exit(xi, err);
1850 atomic_inc_32(&bd->d_kerr->bd_harderrs.value.ui32);
1851
1852 bp->b_resid += xi->i_resid;
1853 bd_xfer_free(xi);
1854 bioerror(bp, err);
1855 biodone(bp);
1856 return;
1857 }
1858
1859 xi->i_cur_win++;
1860 xi->i_resid -= xi->i_len;
1861
1862 if (xi->i_resid == 0) {
1863 /* Job completed succcessfully! */
1864 bd_runq_exit(xi, 0);
1865
1866 bd_xfer_free(xi);
1867 biodone(bp);
1868 return;
1869 }
1870
1871 xi->i_blkno += xi->i_nblks;
1872
1873 if (bd->d_use_dma) {
1874 /* More transfer still pending... advance to next DMA window. */
1875 rv = ddi_dma_getwin(xi->i_dmah, xi->i_cur_win,
1876 &xi->i_offset, &len, &xi->i_dmac, &xi->i_ndmac);
1877 } else {
1878 /* Advance memory window. */
1879 xi->i_kaddr += xi->i_len;
1880 xi->i_offset += xi->i_len;
1881 len = min(bp->b_bcount - xi->i_offset, bd->d_maxxfer);
1882 }
1883
1884
1885 if ((rv != DDI_SUCCESS) ||
1886 (P2PHASE(len, (1U << xi->i_blkshift) != 0))) {
1887 bd_runq_exit(xi, EFAULT);
1888
1889 bp->b_resid += xi->i_resid;
1890 bd_xfer_free(xi);
1891 bioerror(bp, EFAULT);
1892 biodone(bp);
1893 return;
1894 }
1895 xi->i_len = len;
1896 xi->i_nblks = len >> xi->i_blkshift;
1897
1898 /* Submit next window to hardware. */
1899 rv = xi->i_func(bd->d_private, &xi->i_public);
1900 if (rv != 0) {
1901 bd_runq_exit(xi, rv);
1902
1903 atomic_inc_32(&bd->d_kerr->bd_transerrs.value.ui32);
1904
1905 bp->b_resid += xi->i_resid;
1906 bd_xfer_free(xi);
1907 bioerror(bp, rv);
1908 biodone(bp);
1909 }
1910 }
1911
1912 void
1913 bd_error(bd_xfer_t *xfer, int error)
1914 {
1915 bd_xfer_impl_t *xi = (void *)xfer;
1916 bd_t *bd = xi->i_bd;
1917
1918 switch (error) {
1919 case BD_ERR_MEDIA:
1920 atomic_inc_32(&bd->d_kerr->bd_rq_media_err.value.ui32);
1921 break;
1922 case BD_ERR_NTRDY:
1923 atomic_inc_32(&bd->d_kerr->bd_rq_ntrdy_err.value.ui32);
1924 break;
1925 case BD_ERR_NODEV:
1926 atomic_inc_32(&bd->d_kerr->bd_rq_nodev_err.value.ui32);
1927 break;
1928 case BD_ERR_RECOV:
1929 atomic_inc_32(&bd->d_kerr->bd_rq_recov_err.value.ui32);
1930 break;
1931 case BD_ERR_ILLRQ:
1932 atomic_inc_32(&bd->d_kerr->bd_rq_illrq_err.value.ui32);
1933 break;
1934 case BD_ERR_PFA:
1935 atomic_inc_32(&bd->d_kerr->bd_rq_pfa_err.value.ui32);
1936 break;
1937 default:
1938 cmn_err(CE_PANIC, "bd_error: unknown error type %d", error);
1939 break;
1940 }
1941 }
1942
1943 void
1944 bd_state_change(bd_handle_t hdl)
1945 {
1946 bd_t *bd;
1947
1948 if ((bd = hdl->h_bd) != NULL) {
1949 bd_update_state(bd);
1950 }
1951 }
1952
1953 void
1954 bd_mod_init(struct dev_ops *devops)
1955 {
1956 static struct bus_ops bd_bus_ops = {
1957 BUSO_REV, /* busops_rev */
1958 nullbusmap, /* bus_map */
1959 NULL, /* bus_get_intrspec (OBSOLETE) */
1960 NULL, /* bus_add_intrspec (OBSOLETE) */
1961 NULL, /* bus_remove_intrspec (OBSOLETE) */
1962 i_ddi_map_fault, /* bus_map_fault */
1963 NULL, /* bus_dma_map (OBSOLETE) */
1964 ddi_dma_allochdl, /* bus_dma_allochdl */
1965 ddi_dma_freehdl, /* bus_dma_freehdl */
1966 ddi_dma_bindhdl, /* bus_dma_bindhdl */
1967 ddi_dma_unbindhdl, /* bus_dma_unbindhdl */
1968 ddi_dma_flush, /* bus_dma_flush */
1969 ddi_dma_win, /* bus_dma_win */
1970 ddi_dma_mctl, /* bus_dma_ctl */
1971 bd_bus_ctl, /* bus_ctl */
1972 ddi_bus_prop_op, /* bus_prop_op */
1973 NULL, /* bus_get_eventcookie */
1974 NULL, /* bus_add_eventcall */
1975 NULL, /* bus_remove_eventcall */
1976 NULL, /* bus_post_event */
1977 NULL, /* bus_intr_ctl (OBSOLETE) */
1978 NULL, /* bus_config */
1979 NULL, /* bus_unconfig */
1980 NULL, /* bus_fm_init */
1981 NULL, /* bus_fm_fini */
1982 NULL, /* bus_fm_access_enter */
1983 NULL, /* bus_fm_access_exit */
1984 NULL, /* bus_power */
1985 NULL, /* bus_intr_op */
1986 };
1987
1988 devops->devo_bus_ops = &bd_bus_ops;
1989
1990 /*
1991 * NB: The device driver is free to supply its own
1992 * character entry device support.
1993 */
1994 }
1995
1996 void
1997 bd_mod_fini(struct dev_ops *devops)
1998 {
1999 devops->devo_bus_ops = NULL;
2000 }
Unleashed OS