From 481d12aa0b705189346ef8bf87af833f12599914 Mon Sep 17 00:00:00 2001 From: Matthew Dillon Date: Mon, 9 Jan 2017 13:54:05 -0800 Subject: [PATCH] kernel - Incidental MPLOCK removal (non-performance) * Remove the MPLOCK in a ton of places that don't need it or can be trivially tokenized or locked. * Mostly non-performance adjustments, though the X server does use scmouse or sysmouse. --- sys/bus/pci/pci_user.c | 6 +++++- sys/dev/disk/md/md.c | 23 ++++++++++++++++++----- sys/dev/misc/cpuctl/cpuctl.c | 8 +++++++- sys/dev/misc/efirt/efidev.c | 8 +++++++- sys/dev/misc/kbd/kbd.c | 2 +- sys/dev/misc/led/led.c | 2 +- sys/dev/misc/syscons/scmouse.c | 31 ++++++++++++++++--------------- sys/dev/misc/syscons/syscons.c | 2 +- sys/dev/misc/syscons/sysmouse.c | 2 +- sys/dev/serial/sio/sio.c | 3 ++- sys/dev/sound/pcm/mixer.c | 2 +- sys/dev/sound/pcm/sndstat.c | 2 +- sys/kern/kern_udev.c | 3 ++- sys/kern/subr_bus.c | 32 +++++++++++++++++++++++++------- sys/kern/tty_cons.c | 4 ++-- sys/kern/tty_tty.c | 12 +++++++----- sys/platform/pc64/x86_64/vm_machdep.c | 2 +- sys/vfs/nfs/nfs_subs.c | 2 +- 18 files changed, 99 insertions(+), 47 deletions(-) diff --git a/sys/bus/pci/pci_user.c b/sys/bus/pci/pci_user.c index 2e9450ebac..a97148c9c7 100644 --- a/sys/bus/pci/pci_user.c +++ b/sys/bus/pci/pci_user.c @@ -55,6 +55,7 @@ /* * This is the user interface to PCI configuration space. */ +static struct lwkt_token pci_token = LWKT_TOKEN_INITIALIZER(pci_token); static d_open_t pci_open; static d_close_t pci_close; @@ -63,7 +64,7 @@ static int pci_conf_match(struct pci_match_conf *matches, int num_matches, static d_ioctl_t pci_ioctl; struct dev_ops pci_ops = { - { "pci", 0, 0 }, + { "pci", 0, D_MPSAFE }, .d_open = pci_open, .d_close = pci_close, .d_ioctl = pci_ioctl, @@ -321,6 +322,8 @@ pci_ioctl(struct dev_ioctl_args *ap) return EPERM; #endif + lwkt_gettoken(&pci_token); + switch(ap->a_cmd) { #ifdef PRE7_COMPAT case PCIOCGETCONF_OLD: @@ -740,6 +743,7 @@ getconfexit: error = ENOTTY; break; } + lwkt_reltoken(&pci_token); return (error); } diff --git a/sys/dev/disk/md/md.c b/sys/dev/disk/md/md.c index ed75f81dc6..9b4ea63114 100644 --- a/sys/dev/disk/md/md.c +++ b/sys/dev/disk/md/md.c @@ -55,7 +55,7 @@ static d_close_t mdclose; static d_ioctl_t mdioctl; static struct dev_ops md_ops = { - { "md", 0, D_DISK | D_CANFREE | D_MEMDISK | D_TRACKCLOSE}, + { "md", 0, D_DISK | D_CANFREE | D_MEMDISK | D_TRACKCLOSE | D_MPSAFE}, .d_open = mdopen, .d_close = mdclose, .d_read = physread, @@ -65,6 +65,7 @@ static struct dev_ops md_ops = { }; struct md_s { + struct lwkt_token tok; int unit; struct devstat stats; struct bio_queue_head bio_queue; @@ -124,15 +125,18 @@ mdopen(struct dev_open_args *ap) cdev_t dev = ap->a_head.a_dev; struct md_s *sc; - if (md_debug) + if (md_debug) { kprintf("mdopen(%s %x %x)\n", devtoname(dev), ap->a_oflags, ap->a_devtype); + } sc = dev->si_drv1; + lwkt_gettoken(&sc->tok); if (sc->unit + 1 == mdunits) mdcreate_malloc(); - atomic_add_int(&refcnt, 1); + lwkt_reltoken(&sc->tok); + return (0); } @@ -140,11 +144,16 @@ static int mdclose(struct dev_close_args *ap) { cdev_t dev = ap->a_head.a_dev; + struct md_s *sc; - if (md_debug) + if (md_debug) { kprintf("mdclose(%s %x %x)\n", devtoname(dev), ap->a_fflag, ap->a_devtype); + } + sc = dev->si_drv1; + lwkt_gettoken(&sc->tok); atomic_add_int(&refcnt, -1); + lwkt_reltoken(&sc->tok); return (0); } @@ -154,9 +163,10 @@ mdioctl(struct dev_ioctl_args *ap) { cdev_t dev = ap->a_head.a_dev; - if (md_debug) + if (md_debug) { kprintf("mdioctl(%s %lx %p %x)\n", devtoname(dev), ap->a_cmd, ap->a_data, ap->a_fflag); + } return (ENOIOCTL); } @@ -177,11 +187,13 @@ mdstrategy(struct dev_strategy_args *ap) } bio->bio_driver_info = dev; sc = dev->si_drv1; + lwkt_gettoken(&sc->tok); if (sc->type == MD_MALLOC) { mdstrategy_malloc(ap); } else { mdstrategy_preload(ap); } + lwkt_reltoken(&sc->tok); return(0); } @@ -394,6 +406,7 @@ mdcreate(unsigned length) struct disk_info info; sc = kmalloc(sizeof(*sc), M_MD, M_WAITOK | M_ZERO); + lwkt_token_init(&sc->tok, "md"); sc->unit = mdunits++; bioq_init(&sc->bio_queue); devstat_add_entry(&sc->stats, "md", sc->unit, DEV_BSIZE, diff --git a/sys/dev/misc/cpuctl/cpuctl.c b/sys/dev/misc/cpuctl/cpuctl.c index 01e573be9f..d614a8f546 100644 --- a/sys/dev/misc/cpuctl/cpuctl.c +++ b/sys/dev/misc/cpuctl/cpuctl.c @@ -72,11 +72,12 @@ static int update_via(int cpu, cpuctl_update_args_t *args); static cdev_t *cpuctl_devs; static MALLOC_DEFINE(M_CPUCTL, "cpuctl", "CPUCTL buffer"); +static struct lock cpuctl_lock = LOCK_INITIALIZER("cpuctl", 0, 0); static struct dev_ops cpuctl_cdevsw = { + .head = { .name = "cpuctl", .flags = D_MPSAFE }, .d_open = cpuctl_open, .d_ioctl = cpuctl_ioctl, - .head = { .name = "cpuctl" }, }; int @@ -97,6 +98,9 @@ cpuctl_ioctl(struct dev_ioctl_args *ap) cmd == CPUCTL_MSRSBIT || cmd == CPUCTL_MSRCBIT) && ((flags & FWRITE) == 0)) return (EPERM); + + lockmgr(&cpuctl_lock, LK_EXCLUSIVE); + switch (cmd) { case CPUCTL_RDMSR: ret = cpuctl_do_msr(cpu, (cpuctl_msr_args_t *)data, cmd); @@ -128,6 +132,8 @@ cpuctl_ioctl(struct dev_ioctl_args *ap) break; } fail: + lockmgr(&cpuctl_lock, LK_RELEASE); + return (ret); } diff --git a/sys/dev/misc/efirt/efidev.c b/sys/dev/misc/efirt/efidev.c index 62ccba4ffb..549094f033 100644 --- a/sys/dev/misc/efirt/efidev.c +++ b/sys/dev/misc/efirt/efidev.c @@ -39,12 +39,14 @@ #include #include +static struct lock efidev_lock = LOCK_INITIALIZER("efidev", 0, 0); + static d_ioctl_t efidev_ioctl; static d_open_t efidev_open; static d_close_t efidev_close; static struct dev_ops efi_ops = { - { "efi", 0, 0 }, + { "efi", 0, D_MPSAFE }, .d_open = efidev_open, .d_close = efidev_close, .d_ioctl = efidev_ioctl, @@ -69,6 +71,8 @@ efidev_ioctl(struct dev_ioctl_args *ap) caddr_t addr = ap->a_data; int error; + lockmgr(&efidev_lock, LK_EXCLUSIVE); + switch (cmd) { case EFIIOC_GET_TABLE: { @@ -185,6 +189,8 @@ vs_out: break; } + lockmgr(&efidev_lock, LK_RELEASE); + return (error); } diff --git a/sys/dev/misc/kbd/kbd.c b/sys/dev/misc/kbd/kbd.c index 87eadfd3e4..04619fcafa 100644 --- a/sys/dev/misc/kbd/kbd.c +++ b/sys/dev/misc/kbd/kbd.c @@ -582,7 +582,7 @@ static void genkbdfiltdetach(struct knote *); static int genkbdfilter(struct knote *, long); static struct dev_ops kbd_ops = { - { "kbd", 0, 0 }, + { "kbd", 0, D_MPSAFE }, .d_open = genkbdopen, .d_close = genkbdclose, .d_read = genkbdread, diff --git a/sys/dev/misc/led/led.c b/sys/dev/misc/led/led.c index 5230a055f7..eb007e224b 100644 --- a/sys/dev/misc/led/led.c +++ b/sys/dev/misc/led/led.c @@ -277,7 +277,7 @@ led_set(char const *name, char const *cmd) } static struct dev_ops led_ops = { - { "LED", 0, 0 }, + { "LED", 0, D_MPSAFE }, .d_open = led_open, .d_close = led_close, .d_write = led_write, diff --git a/sys/dev/misc/syscons/scmouse.c b/sys/dev/misc/syscons/scmouse.c index 7bae4f6515..238fe547bd 100644 --- a/sys/dev/misc/syscons/scmouse.c +++ b/sys/dev/misc/syscons/scmouse.c @@ -36,7 +36,6 @@ #include #include #include -#include #include #include @@ -75,6 +74,8 @@ static void mouse_cut_extend(scr_stat *scp); static void mouse_paste(scr_stat *scp); #endif /* SC_NO_CUTPASTE */ +static struct lwkt_token scmouse_tok = LWKT_TOKEN_INITIALIZER(scmouse_tok); + #ifndef SC_NO_CUTPASTE /* allocate a cut buffer */ void @@ -581,14 +582,14 @@ sc_mouse_exit1_proc(struct proc *p) scp = p->p_drv_priv; KKASSERT(scp != NULL); - get_mplock(); + lwkt_gettoken(&scmouse_tok); KKASSERT(scp->mouse_proc == p); KKASSERT(scp->mouse_pid == p->p_pid); scp->mouse_signal = 0; scp->mouse_proc = NULL; scp->mouse_pid = 0; - rel_mplock(); + lwkt_reltoken(&scmouse_tok); PRELE(p); p->p_flags &= ~P_SCMOUSE; @@ -638,28 +639,28 @@ sc_mouse_ioctl(struct tty *tp, u_long cmd, caddr_t data, int flag) * Setup a process to receive signals on mouse events. */ case MOUSE_MODE: - get_mplock(); + lwkt_gettoken(&scmouse_tok); if (!ISSIGVALID(mouse->u.mode.signal)) { /* Setting MOUSE_MODE w/ an invalid signal is used to disarm */ if (scp->mouse_proc == curproc) { sc_mouse_exit1_proc(curproc); - rel_mplock(); + lwkt_reltoken(&scmouse_tok); return 0; } else { - rel_mplock(); + lwkt_reltoken(&scmouse_tok); return EINVAL; } } else { /* Only one mouse process per syscons */ if (scp->mouse_proc) { - rel_mplock(); + lwkt_reltoken(&scmouse_tok); return EINVAL; } /* Only one syscons signal source per process */ if (curproc->p_flags & P_SCMOUSE) { - rel_mplock(); + lwkt_reltoken(&scmouse_tok); return EINVAL; } @@ -676,7 +677,7 @@ sc_mouse_ioctl(struct tty *tp, u_long cmd, caddr_t data, int flag) curproc->p_drv_priv = scp; PHOLD(curproc); - rel_mplock(); + lwkt_reltoken(&scmouse_tok); return 0; } /*NOTREACHED*/ @@ -770,14 +771,14 @@ sc_mouse_ioctl(struct tty *tp, u_long cmd, caddr_t data, int flag) cur_scp->status &= ~MOUSE_HIDDEN; - get_mplock(); + lwkt_gettoken(&scmouse_tok); if (cur_scp->mouse_signal) { KKASSERT(cur_scp->mouse_proc != NULL); ksignal(cur_scp->mouse_proc, cur_scp->mouse_signal); - rel_mplock(); + lwkt_reltoken(&scmouse_tok); break; } - rel_mplock(); + lwkt_reltoken(&scmouse_tok); if (ISGRAPHSC(cur_scp) || (cut_buffer == NULL)) break; @@ -820,14 +821,14 @@ sc_mouse_ioctl(struct tty *tp, u_long cmd, caddr_t data, int flag) cur_scp->status &= ~MOUSE_HIDDEN; - get_mplock(); + lwkt_gettoken(&scmouse_tok); if (cur_scp->mouse_signal) { KKASSERT(cur_scp->mouse_proc != NULL); ksignal(cur_scp->mouse_proc, cur_scp->mouse_signal); - rel_mplock(); + lwkt_reltoken(&scmouse_tok); break; } - rel_mplock(); + lwkt_reltoken(&scmouse_tok); if (ISGRAPHSC(cur_scp) || (cut_buffer == NULL)) break; diff --git a/sys/dev/misc/syscons/syscons.c b/sys/dev/misc/syscons/syscons.c index c0af11ad6d..c26c6cee1a 100644 --- a/sys/dev/misc/syscons/syscons.c +++ b/sys/dev/misc/syscons/syscons.c @@ -285,7 +285,7 @@ static d_ioctl_t scioctl; static d_mmap_t scmmap; static struct dev_ops sc_ops = { - { "sc", 0, D_TTY }, + { "sc", 0, D_TTY | D_MPSAFE }, .d_open = scopen, .d_close = scclose, .d_read = scread, diff --git a/sys/dev/misc/syscons/sysmouse.c b/sys/dev/misc/syscons/sysmouse.c index b14514422c..3a8c3bae79 100644 --- a/sys/dev/misc/syscons/sysmouse.c +++ b/sys/dev/misc/syscons/sysmouse.c @@ -57,7 +57,7 @@ static d_close_t smclose; static d_ioctl_t smioctl; static struct dev_ops sm_ops = { - { "sysmouse", 0, D_TTY }, + { "sysmouse", 0, D_TTY | D_MPSAFE }, .d_open = smopen, .d_close = smclose, .d_read = ttyread, diff --git a/sys/dev/serial/sio/sio.c b/sys/dev/serial/sio/sio.c index e34ba28d71..5adc987aac 100644 --- a/sys/dev/serial/sio/sio.c +++ b/sys/dev/serial/sio/sio.c @@ -1187,7 +1187,8 @@ determined_type: ; kprintf("\n"); if (!sio_registered) { - register_swi(SWI_TTY, siopoll, NULL ,"swi_siopoll", NULL, -1); + register_swi_mp(SWI_TTY, siopoll, NULL, + "swi_siopoll", NULL, -1); sio_registered = TRUE; } minorbase = UNIT_TO_MINOR(unit); diff --git a/sys/dev/sound/pcm/mixer.c b/sys/dev/sound/pcm/mixer.c index d7f9af8770..0b26830486 100644 --- a/sys/dev/sound/pcm/mixer.c +++ b/sys/dev/sound/pcm/mixer.c @@ -101,7 +101,7 @@ static d_close_t mixer_close; static d_ioctl_t mixer_ioctl; struct dev_ops mixer_ops = { - { "mixer", 0, D_TRACKCLOSE }, + { "mixer", 0, D_TRACKCLOSE | D_MPSAFE }, .d_open = mixer_open, .d_close = mixer_close, .d_ioctl = mixer_ioctl, diff --git a/sys/dev/sound/pcm/sndstat.c b/sys/dev/sound/pcm/sndstat.c index 3602644aaa..a70fa77628 100644 --- a/sys/dev/sound/pcm/sndstat.c +++ b/sys/dev/sound/pcm/sndstat.c @@ -48,7 +48,7 @@ static d_close_t sndstat_close; static d_read_t sndstat_read; static struct dev_ops sndstat_cdevsw = { - { "sndstat", 0, 0 }, + { "sndstat", 0, D_MPSAFE }, .d_open = sndstat_open, .d_close = sndstat_close, .d_read = sndstat_read, diff --git a/sys/kern/kern_udev.c b/sys/kern/kern_udev.c index 24923ffaa4..afa706db44 100644 --- a/sys/kern/kern_udev.c +++ b/sys/kern/kern_udev.c @@ -648,7 +648,8 @@ udev_dev_close(struct dev_close_args *ap) } static struct filterops udev_dev_read_filtops = - { FILTEROP_ISFD, NULL, udev_dev_filter_detach, udev_dev_filter_read }; + { FILTEROP_ISFD | FILTEROP_MPSAFE, NULL, + udev_dev_filter_detach, udev_dev_filter_read }; static int udev_dev_kqfilter(struct dev_kqfilter_args *ap) diff --git a/sys/kern/subr_bus.c b/sys/kern/subr_bus.c index 6565a744dd..46c17b9bc3 100644 --- a/sys/kern/subr_bus.c +++ b/sys/kern/subr_bus.c @@ -52,7 +52,6 @@ #include /* for device_printf() */ #include -#include SYSCTL_NODE(_hw, OID_AUTO, bus, CTLFLAG_RW, NULL, NULL); SYSCTL_NODE(, OID_AUTO, dev, CTLFLAG_RW, NULL, NULL); @@ -282,7 +281,7 @@ static d_ioctl_t devioctl; static d_kqfilter_t devkqfilter; static struct dev_ops devctl_ops = { - { "devctl", 0, 0 }, + { "devctl", 0, D_MPSAFE }, .d_open = devopen, .d_close = devclose, .d_read = devread, @@ -307,30 +306,48 @@ static struct dev_softc struct proc *async_proc; } devsoftc; +/* + * Chicken-and-egg problem with devfs, get the queue operational early. + */ static void -devinit(void) +predevinit(void) { - make_dev(&devctl_ops, 0, UID_ROOT, GID_WHEEL, 0600, "devctl"); lockinit(&devsoftc.lock, "dev mtx", 0, 0); TAILQ_INIT(&devsoftc.devq); } +SYSINIT(predevinit, SI_SUB_CREATE_INIT, SI_ORDER_ANY, predevinit, 0); + +static void +devinit(void) +{ + /* + * WARNING! make_dev() can call back into devctl_queue_data() + * immediately. + */ + make_dev(&devctl_ops, 0, UID_ROOT, GID_WHEEL, 0600, "devctl"); +} static int devopen(struct dev_open_args *ap) { - if (devsoftc.inuse) + lockmgr(&devsoftc.lock, LK_EXCLUSIVE); + if (devsoftc.inuse) { + lockmgr(&devsoftc.lock, LK_RELEASE); return (EBUSY); + } /* move to init */ devsoftc.inuse = 1; devsoftc.async_proc = NULL; + lockmgr(&devsoftc.lock, LK_RELEASE); + return (0); } static int devclose(struct dev_close_args *ap) { - devsoftc.inuse = 0; lockmgr(&devsoftc.lock, LK_EXCLUSIVE); + devsoftc.inuse = 0; wakeup(&devsoftc); lockmgr(&devsoftc.lock, LK_RELEASE); @@ -409,7 +426,8 @@ static void dev_filter_detach(struct knote *); static int dev_filter_read(struct knote *, long); static struct filterops dev_filtops = - { FILTEROP_ISFD, NULL, dev_filter_detach, dev_filter_read }; + { FILTEROP_ISFD | FILTEROP_MPSAFE, NULL, + dev_filter_detach, dev_filter_read }; static int devkqfilter(struct dev_kqfilter_args *ap) diff --git a/sys/kern/tty_cons.c b/sys/kern/tty_cons.c index 149d056abd..00f4b2c846 100644 --- a/sys/kern/tty_cons.c +++ b/sys/kern/tty_cons.c @@ -68,7 +68,7 @@ static int cnintercept(struct dev_generic_args *ap); #define CDEV_MAJOR 0 static struct dev_ops cn_ops = { - { "console", 0, D_TTY }, + { "console", 0, D_TTY | D_MPSAFE }, .d_open = cnopen, .d_close = cnclose, .d_read = cnread, @@ -78,7 +78,7 @@ static struct dev_ops cn_ops = { }; static struct dev_ops cn_iops = { - { "intercept", 0, D_TTY }, + { "intercept", 0, D_TTY | D_MPSAFE }, .d_default = cnintercept }; diff --git a/sys/kern/tty_tty.c b/sys/kern/tty_tty.c index de72f32d5f..a226a48812 100644 --- a/sys/kern/tty_tty.c +++ b/sys/kern/tty_tty.c @@ -62,7 +62,7 @@ static int cttyfilt_write(struct knote *, long); #define CDEV_MAJOR 1 static struct dev_ops ctty_ops = { - { "ctty", 0, D_TTY }, + { "ctty", 0, D_TTY | D_MPSAFE }, .d_open = cttyopen, .d_close = cttyclose, .d_read = cttyread, @@ -263,9 +263,11 @@ cttyioctl(struct dev_ioctl_args *ap) } static struct filterops cttyfiltops_read = - { FILTEROP_ISFD, NULL, cttyfilt_detach, cttyfilt_read }; + { FILTEROP_ISFD | FILTEROP_MPSAFE, NULL, + cttyfilt_detach, cttyfilt_read }; static struct filterops cttyfiltops_write = - { FILTEROP_ISFD, NULL, cttyfilt_detach, cttyfilt_write }; + { FILTEROP_ISFD | FILTEROP_MPSAFE, NULL, + cttyfilt_detach, cttyfilt_write }; static int cttykqfilter(struct dev_kqfilter_args *ap) @@ -331,5 +333,5 @@ ctty_drvinit(void *unused __unused) make_dev(&ctty_ops, 0, 0, 0, 0666, "tty"); } -SYSINIT(cttydev, SI_SUB_DRIVERS, SI_ORDER_MIDDLE + CDEV_MAJOR, ctty_drvinit, - NULL); +SYSINIT(cttydev, SI_SUB_DRIVERS, SI_ORDER_MIDDLE + CDEV_MAJOR, + ctty_drvinit, NULL); diff --git a/sys/platform/pc64/x86_64/vm_machdep.c b/sys/platform/pc64/x86_64/vm_machdep.c index 9f68cc6fa6..8c3f418ac4 100644 --- a/sys/platform/pc64/x86_64/vm_machdep.c +++ b/sys/platform/pc64/x86_64/vm_machdep.c @@ -358,7 +358,7 @@ swi_vm(void *arg, void *frame) static void swi_vm_setup(void *arg) { - register_swi(SWI_VM, swi_vm, NULL, "swi_vm", NULL, 0); + register_swi_mp(SWI_VM, swi_vm, NULL, "swi_vm", NULL, 0); } SYSINIT(vm_setup, SI_BOOT2_MACHDEP, SI_ORDER_ANY, swi_vm_setup, NULL); diff --git a/sys/vfs/nfs/nfs_subs.c b/sys/vfs/nfs/nfs_subs.c index 7106c7db62..dfb260fc2a 100644 --- a/sys/vfs/nfs/nfs_subs.c +++ b/sys/vfs/nfs/nfs_subs.c @@ -566,7 +566,7 @@ nfs_curusec(void) int nfs_init(struct vfsconf *vfsp) { - callout_init(&nfs_timer_handle); + callout_init_mp(&nfs_timer_handle); nfsmount_objcache = objcache_create_simple(M_NFSMOUNT, sizeof(struct nfsmount)); nfs_mount_type = vfsp->vfc_typenum; -- 2.11.4.GIT