kernel/vm/seg_umap.c

   1 /*
   2  * This file and its contents are supplied under the terms of the
   3  * Common Development and Distribution License ("CDDL"), version 1.0.
   4  * You may only use this file in accordance with the terms of version
   5  * 1.0 of the CDDL.
   6  *
   7  * A full copy of the text of the CDDL should have accompanied this
   8  * source.  A copy of the CDDL is also available via the Internet at
   9  * http://www.illumos.org/license/CDDL.
  10  */
  11
  12 /*
  13  * Copyright 2018 Joyent, Inc.
  14  */
  15
  16 /*
  17  * VM - Kernel-to-user mapping segment
  18  *
  19  * The umap segment driver was primarily designed to facilitate the comm page:
  20  * a portion of kernel memory shared with userspace so that certain (namely
  21  * clock-related) actions could operate without making an expensive trip into
  22  * the kernel.
  23  *
  24  * Since the initial requirements for the comm page are slim, advanced features
  25  * of the segment driver such as per-page protection have been left
  26  * unimplemented at this time.
  27  */
  28
  29
  30 #include <sys/types.h>
  31 #include <sys/param.h>
  32 #include <sys/errno.h>
  33 #include <sys/cred.h>
  34 #include <sys/kmem.h>
  35 #include <sys/lgrp.h>
  36 #include <sys/mman.h>
  37
  38 #include <vm/hat.h>
  39 #include <vm/as.h>
  40 #include <vm/seg.h>
  41 #include <vm/seg_kmem.h>
  42 #include <vm/seg_umap.h>
  43
  44
  45 static boolean_t segumap_verify_safe(caddr_t, size_t);
  46 static int segumap_dup(struct seg *, struct seg *);
  47 static int segumap_unmap(struct seg *, caddr_t, size_t);
  48 static void segumap_free(struct seg *);
  49 static faultcode_t segumap_fault(struct hat *, struct seg *, caddr_t, size_t,
  50     enum fault_type, enum seg_rw);
  51 static faultcode_t segumap_faulta(struct seg *, caddr_t);
  52 static int segumap_setprot(struct seg *, caddr_t, size_t, uint_t);
  53 static int segumap_checkprot(struct seg *, caddr_t, size_t, uint_t);
  54 static int segumap_sync(struct seg *, caddr_t, size_t, int, uint_t);
  55 static size_t segumap_incore(struct seg *, caddr_t, size_t, char *);
  56 static int segumap_lockop(struct seg *, caddr_t, size_t, int, int, ulong_t *,
  57     size_t);
  58 static int segumap_getprot(struct seg *, caddr_t, size_t, uint_t *);
  59 static uoff_t segumap_getoffset(struct seg *, caddr_t);
  60 static int segumap_gettype(struct seg *, caddr_t);
  61 static int segumap_getvp(struct seg *, caddr_t, struct vnode **);
  62 static int segumap_advise(struct seg *, caddr_t, size_t, uint_t);
  63 static void segumap_dump(struct seg *);
  64 static int segumap_pagelock(struct seg *, caddr_t, size_t, struct page ***,
  65     enum lock_type, enum seg_rw);
  66 static int segumap_setpagesize(struct seg *, caddr_t, size_t, uint_t);
  67 static int segumap_getmemid(struct seg *, caddr_t, memid_t *);
  68 static int segumap_capable(struct seg *, segcapability_t);
  69
  70 static struct seg_ops segumap_ops = {
  71         .dup            = segumap_dup,
  72         .unmap          = segumap_unmap,
  73         .free           = segumap_free,
  74         .fault          = segumap_fault,
  75         .faulta         = segumap_faulta,
  76         .setprot        = segumap_setprot,
  77         .checkprot      = segumap_checkprot,
  78         .sync           = segumap_sync,
  79         .incore         = segumap_incore,
  80         .lockop         = segumap_lockop,
  81         .getprot        = segumap_getprot,
  82         .getoffset      = segumap_getoffset,
  83         .gettype        = segumap_gettype,
  84         .getvp          = segumap_getvp,
  85         .advise         = segumap_advise,
  86         .dump           = segumap_dump,
  87         .pagelock       = segumap_pagelock,
  88         .setpagesize    = segumap_setpagesize,
  89         .getmemid       = segumap_getmemid,
  90         .capable        = segumap_capable,
  91 };
  92
  93
  94 /*
  95  * Create a kernel/user-mapped segment.
  96  */
  97 int
  98 segumap_create(struct seg **segpp, void *argsp)
  99 {
 100         struct seg *seg = *segpp;
 101         segumap_crargs_t *a = (struct segumap_crargs *)argsp;
 102         segumap_data_t *data;
 103
 104         ASSERT((uintptr_t)a->kaddr > _userlimit);
 105
 106         /*
 107          * Check several aspects of the mapping request to ensure validity:
 108          * - kernel pages must reside entirely in kernel space
 109          * - target protection must be user-accessible
 110          * - kernel address must be page-aligned
 111          * - kernel address must reside inside a "safe" segment
 112          */
 113         if ((uintptr_t)a->kaddr <= _userlimit ||
 114             ((uintptr_t)a->kaddr + seg->s_size) < (uintptr_t)a->kaddr ||
 115             (a->prot & PROT_USER) == 0 ||
 116             ((uintptr_t)a->kaddr & PAGEOFFSET) != 0 ||
 117             !segumap_verify_safe(a->kaddr, seg->s_size)) {
 118                 return (EINVAL);
 119         }
 120
 121         data = kmem_zalloc(sizeof (*data), KM_SLEEP);
 122         rw_init(&data->sud_lock, NULL, RW_DEFAULT, NULL);
 123         data->sud_kaddr = a->kaddr;
 124         data->sud_prot = a->prot;
 125
 126         seg->s_ops = &segumap_ops;
 127         seg->s_data = data;
 128         return (0);
 129 }
 130
 131 static boolean_t
 132 segumap_verify_safe(caddr_t kaddr, size_t len)
 133 {
 134         struct seg *seg;
 135
 136         /*
 137          * Presently, only pages which are backed by segkmem are allowed to be
 138          * shared with userspace.  This prevents nasty paging behavior with
 139          * other drivers such as seg_kp.  Furthermore, the backing kernel
 140          * segment must completely contain the region to be mapped.
 141          *
 142          * Failing these checks is fatal for now since such mappings are done
 143          * in a very limited context from the kernel.
 144          */
 145         AS_LOCK_ENTER(&kas, RW_READER);
 146         seg = as_segat(&kas, kaddr);
 147         VERIFY(seg != NULL);
 148         VERIFY(seg->s_base + seg->s_size >= kaddr + len);
 149         VERIFY(seg->s_ops == &segkmem_ops);
 150         AS_LOCK_EXIT(&kas);
 151
 152         return (B_TRUE);
 153 }
 154
 155 static int
 156 segumap_dup(struct seg *seg, struct seg *newseg)
 157 {
 158         segumap_data_t *sud = (segumap_data_t *)seg->s_data;
 159         segumap_data_t *newsud;
 160
 161         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
 162
 163         newsud = kmem_zalloc(sizeof (segumap_data_t), KM_SLEEP);
 164         rw_init(&newsud->sud_lock, NULL, RW_DEFAULT, NULL);
 165         newsud->sud_kaddr = sud->sud_kaddr;
 166         newsud->sud_prot = sud->sud_prot;
 167
 168         newseg->s_ops = seg->s_ops;
 169         newseg->s_data = newsud;
 170         return (0);
 171 }
 172
 173 static int
 174 segumap_unmap(struct seg *seg, caddr_t addr, size_t len)
 175 {
 176         segumap_data_t *sud = (segumap_data_t *)seg->s_data;
 177
 178         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as));
 179
 180         /* Only allow unmap of entire segment */
 181         if (addr != seg->s_base || len != seg->s_size) {
 182                 return (EINVAL);
 183         }
 184         if (sud->sud_softlockcnt != 0) {
 185                 return (EAGAIN);
 186         }
 187
 188         /*
 189          * Unconditionally unload the entire segment range.
 190          */
 191         hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
 192
 193         seg_free(seg);
 194         return (0);
 195 }
 196
 197 static void
 198 segumap_free(struct seg *seg)
 199 {
 200         segumap_data_t *data = (segumap_data_t *)seg->s_data;
 201
 202         ASSERT(data != NULL);
 203
 204         rw_destroy(&data->sud_lock);
 205         VERIFY(data->sud_softlockcnt == 0);
 206         kmem_free(data, sizeof (*data));
 207         seg->s_data = NULL;
 208 }
 209
 210 /* ARGSUSED */
 211 static faultcode_t
 212 segumap_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t len,
 213     enum fault_type type, enum seg_rw tw)
 214 {
 215         segumap_data_t *sud = (segumap_data_t *)seg->s_data;
 216
 217         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
 218
 219         if (type == F_PROT) {
 220                 /*
 221                  * Since protection on the segment is fixed, there is nothing
 222                  * to do but report an error for protection faults.
 223                  */
 224                 return (FC_PROT);
 225         } else if (type == F_SOFTUNLOCK) {
 226                 size_t plen = btop(len);
 227
 228                 rw_enter(&sud->sud_lock, RW_WRITER);
 229                 VERIFY(sud->sud_softlockcnt >= plen);
 230                 sud->sud_softlockcnt -= plen;
 231                 rw_exit(&sud->sud_lock);
 232                 return (0);
 233         }
 234
 235         ASSERT(type == F_INVAL || type == F_SOFTLOCK);
 236         rw_enter(&sud->sud_lock, RW_WRITER);
 237
 238         if (type == F_INVAL ||
 239             (type == F_SOFTLOCK && sud->sud_softlockcnt == 0)) {
 240                 /*
 241                  * Load the (entire) segment into the HAT.
 242                  *
 243                  * It's possible that threads racing into as_fault will cause
 244                  * seg_umap to load the same range multiple times in quick
 245                  * succession.  Redundant hat_devload operations are safe.
 246                  */
 247                 for (uintptr_t i = 0; i < seg->s_size; i += PAGESIZE) {
 248                         pfn_t pfn;
 249
 250                         pfn = hat_getpfnum(kas.a_hat, sud->sud_kaddr + i);
 251                         VERIFY(pfn != PFN_INVALID);
 252                         hat_devload(seg->s_as->a_hat, seg->s_base + i,
 253                             PAGESIZE, pfn, sud->sud_prot, HAT_LOAD);
 254                 }
 255         }
 256         if (type == F_SOFTLOCK) {
 257                 size_t nval = sud->sud_softlockcnt + btop(len);
 258
 259                 if (sud->sud_softlockcnt >= nval) {
 260                         rw_exit(&sud->sud_lock);
 261                         return (FC_MAKE_ERR(EOVERFLOW));
 262                 }
 263                 sud->sud_softlockcnt = nval;
 264         }
 265
 266         rw_exit(&sud->sud_lock);
 267         return (0);
 268 }
 269
 270 /* ARGSUSED */
 271 static faultcode_t
 272 segumap_faulta(struct seg *seg, caddr_t addr)
 273 {
 274         /* Do nothing since asynch pagefault should not load translation. */
 275         return (0);
 276 }
 277
 278 /* ARGSUSED */
 279 static int
 280 segumap_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
 281 {
 282         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
 283
 284         /*
 285          * The seg_umap driver does not yet allow protection to be changed.
 286          */
 287         return (EACCES);
 288 }
 289
 290 /* ARGSUSED */
 291 static int
 292 segumap_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
 293 {
 294         segumap_data_t *sud = (segumap_data_t *)seg->s_data;
 295         int error = 0;
 296
 297         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
 298
 299         rw_enter(&sud->sud_lock, RW_READER);
 300         if ((sud->sud_prot & prot) != prot) {
 301                 error = EACCES;
 302         }
 303         rw_exit(&sud->sud_lock);
 304         return (error);
 305 }
 306
 307 /* ARGSUSED */
 308 static int
 309 segumap_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
 310 {
 311         /* Always succeed since there are no backing store to sync */
 312         return (0);
 313 }
 314
 315 /* ARGSUSED */
 316 static size_t
 317 segumap_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
 318 {
 319         size_t sz = 0;
 320
 321         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
 322
 323         len = (len + PAGEOFFSET) & PAGEMASK;
 324         while (len > 0) {
 325                 *vec = 1;
 326                 sz += PAGESIZE;
 327                 vec++;
 328                 len -= PAGESIZE;
 329         }
 330         return (sz);
 331 }
 332
 333 /* ARGSUSED */
 334 static int
 335 segumap_lockop(struct seg *seg, caddr_t addr, size_t len, int attr, int op,
 336     ulong_t *lockmap, size_t pos)
 337 {
 338         /* Report success since kernel pages are always in memory. */
 339         return (0);
 340 }
 341
 342 static int
 343 segumap_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
 344 {
 345         segumap_data_t *sud = (segumap_data_t *)seg->s_data;
 346         size_t pgno;
 347         uint_t prot;
 348
 349         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
 350
 351         rw_enter(&sud->sud_lock, RW_READER);
 352         prot = sud->sud_prot;
 353         rw_exit(&sud->sud_lock);
 354
 355         /*
 356          * Reporting protection is simple since it is not tracked per-page.
 357          */
 358         pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
 359         while (pgno > 0) {
 360                 protv[--pgno] = prot;
 361         }
 362         return (0);
 363 }
 364
 365 /* ARGSUSED */
 366 static uoff_t
 367 segumap_getoffset(struct seg *seg, caddr_t addr)
 368 {
 369         /*
 370          * To avoid leaking information about the layout of the kernel address
 371          * space, always report '0' as the offset.
 372          */
 373         return (0);
 374 }
 375
 376 /* ARGSUSED */
 377 static int
 378 segumap_gettype(struct seg *seg, caddr_t addr)
 379 {
 380         /*
 381          * Since already-existing kernel pages are being mapped into userspace,
 382          * always report the segment type as shared.
 383          */
 384         return (MAP_SHARED);
 385 }
 386
 387 /* ARGSUSED */
 388 static int
 389 segumap_getvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
 390 {
 391         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as));
 392
 393         *vpp = NULL;
 394         return (0);
 395 }
 396
 397 /* ARGSUSED */
 398 static int
 399 segumap_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
 400 {
 401         if (behav == MADV_PURGE) {
 402                 /* Purge does not make sense for this mapping */
 403                 return (EINVAL);
 404         }
 405         /* Indicate success for everything else. */
 406         return (0);
 407 }
 408
 409 /* ARGSUSED */
 410 static void
 411 segumap_dump(struct seg *seg)
 412 {
 413         /*
 414          * Since this is a mapping to share kernel data with userspace, nothing
 415          * additional should be dumped.
 416          */
 417 }
 418
 419 /* ARGSUSED */
 420 static int
 421 segumap_pagelock(struct seg *seg, caddr_t addr, size_t len, struct page ***ppp,
 422     enum lock_type type, enum seg_rw rw)
 423 {
 424         return (ENOTSUP);
 425 }
 426
 427 /* ARGSUSED */
 428 static int
 429 segumap_setpagesize(struct seg *seg, caddr_t addr, size_t len, uint_t szc)
 430 {
 431         return (ENOTSUP);
 432 }
 433
 434 static int
 435 segumap_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
 436 {
 437         segumap_data_t *sud = (segumap_data_t *)seg->s_data;
 438
 439         memidp->val[0] = (uintptr_t)sud->sud_kaddr;
 440         memidp->val[1] = (uintptr_t)(addr - seg->s_base);
 441         return (0);
 442 }
 443
 444 /* ARGSUSED */
 445 static int
 446 segumap_capable(struct seg *seg, segcapability_t capability)
 447 {
 448         /* no special capablities */
 449         return (0);
 450 }