sys/sys/globaldata.h

   1 /*
   2  * Copyright (c) 2003-2011 The DragonFly Project.  All rights reserved.
   3  *
   4  * This code is derived from software contributed to The DragonFly Project
   5  * by Matthew Dillon <dillon@backplane.com>
   6  *
   7  * Redistribution and use in source and binary forms, with or without
   8  * modification, are permitted provided that the following conditions
   9  * are met:
  10  *
  11  * 1. Redistributions of source code must retain the above copyright
  12  *    notice, this list of conditions and the following disclaimer.
  13  * 2. Redistributions in binary form must reproduce the above copyright
  14  *    notice, this list of conditions and the following disclaimer in
  15  *    the documentation and/or other materials provided with the
  16  *    distribution.
  17  * 3. Neither the name of The DragonFly Project nor the names of its
  18  *    contributors may be used to endorse or promote products derived
  19  *    from this software without specific, prior written permission.
  20  *
  21  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  22  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  23  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
  24  * FOR A PARTICULAR PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE
  25  * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
  26  * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING,
  27  * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
  28  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
  29  * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
  30  * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
  31  * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  32  * SUCH DAMAGE.
  33  *
  34  * Copyright (c) Peter Wemm <peter@netplex.com.au> All rights reserved.
  35  *
  36  * Redistribution and use in source and binary forms, with or without
  37  * modification, are permitted provided that the following conditions
  38  * are met:
  39  * 1. Redistributions of source code must retain the above copyright
  40  *    notice, this list of conditions and the following disclaimer.
  41  * 2. Redistributions in binary form must reproduce the above copyright
  42  *    notice, this list of conditions and the following disclaimer in the
  43  *    documentation and/or other materials provided with the distribution.
  44  *
  45  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
  46  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
  47  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
  48  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
  49  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
  50  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
  51  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
  52  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
  53  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
  54  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  55  * SUCH DAMAGE.
  56  */
  57
  58 #ifndef _SYS_GLOBALDATA_H_
  59 #define _SYS_GLOBALDATA_H_
  60
  61 #if defined(_KERNEL) || defined(_KERNEL_STRUCTURES)
  62
  63 #ifndef _SYS_TIME_H_
  64 #include <sys/time.h>           /* struct timeval */
  65 #endif
  66 #ifndef _SYS_VMMETER_H_
  67 #include <sys/vmmeter.h>        /* struct vmmeter, pcpu vmstats adj */
  68 #endif
  69 #ifndef _SYS_THREAD_H_
  70 #include <sys/thread.h>         /* struct thread */
  71 #endif
  72 #ifndef _SYS_SLABALLOC_H_
  73 #include <sys/slaballoc.h>      /* SLGlobalData */
  74 #endif
  75 #ifndef _SYS_SYSTIMER_H_
  76 #include <sys/systimer.h>       /* fine-grained system timers */
  77 #endif
  78 #ifndef _SYS_NCHSTATS_H_
  79 #include <sys/nchstats.h>
  80 #endif
  81 #ifndef _SYS_SYSID_H_
  82 #include <sys/sysid.h>          /* sysid_t */
  83 #endif
  84 #ifndef _SYS_CALLOUT_H_
  85 #include <sys/callout.h>
  86 #endif
  87 #ifndef _SYS_INDEFINITE_H_
  88 #include <sys/indefinite.h>
  89 #endif
  90 #ifndef _SYS_LOCK_H_
  91 #include <sys/lock.h>
  92 #endif
  93 #include <machine/stdint.h>
  94
  95 /*
  96  * This structure maps out the global data that needs to be kept on a
  97  * per-cpu basis.  genassym uses this to generate offsets for the assembler
  98  * code.  The machine-dependant portions of this file can be found in
  99  * <machine/globaldata.h>, but only MD code should retrieve it.
 100  *
 101  * The SMP parts are setup in pmap.c and locore.s for the BSP, and
 102  * mp_machdep.c sets up the data for the AP's to "see" when they awake.
 103  * The reason for doing it via a struct is so that an array of pointers
 104  * to each CPU's data can be set up for things like "check curproc on all
 105  * other processors"
 106  *
 107  * NOTE! this structure needs to remain compatible between module accessors
 108  * and the kernel, so we can't throw in lots of #ifdef's.
 109  *
 110  * gd_reqflags serves serveral purposes, but it is primarily an interrupt
 111  * rollup flag used by the task switcher and spl mechanisms to decide that
 112  * further checks are necessary.  Interrupts are typically managed on a
 113  * per-processor basis at least until you leave a critical section, but
 114  * may then be scheduled to other cpus.
 115  *
 116  * gd_vme_avail and gd_vme_base cache free vm_map_entry structures for use
 117  * in various vm_map related operations.  gd_vme_avail is *NOT* a count of
 118  * the number of structures in the cache but is instead a count of the number
 119  * of unreserved structures in the cache.  See vm_map_entry_reserve().
 120  */
 121
 122 struct sysmsg;
 123 struct tslpque;
 124 struct privatespace;
 125 struct vm_map_entry;
 126 struct spinlock;
 127 struct pipe;
 128
 129 struct globaldata {
 130         struct privatespace *gd_prvspace;       /* self-reference */
 131         struct thread   *gd_curthread;
 132         struct thread   *gd_freetd;             /* cache one free td */
 133         __uint32_t      gd_reqflags;            /* (see note above) */
 134         long            gd_flags;
 135         lwkt_queue      gd_tdallq;              /* all threads */
 136         lwkt_queue      gd_tdrunq;              /* runnable threads */
 137         __uint32_t      gd_cpuid;
 138         cpumask_t       gd_cpumask;             /* CPUMASK_ASSBIT(cpuid) */
 139         cpumask_t       gd_other_cpus;          /* mask of 'other' cpus */
 140         union {
 141                 struct timeval  gd_stattv;
 142                 sysclock_t      gd_statcv;
 143         } statint;
 144         int             gd_intr_nesting_level;  /* hard code, intrs, ipis */
 145         struct vmmeter  gd_cnt;
 146         struct vmtotal  gd_vmtotal;
 147         cpumask_t       gd_ipimask;             /* pending ipis from cpus */
 148         struct lwkt_ipiq *gd_ipiq;              /* array[ncpu] of ipiq's */
 149         struct lwkt_ipiq gd_cpusyncq;           /* ipiq for cpu synchro */
 150         u_int           gd_npoll;               /* ipiq synchronization */
 151         int             gd_tdrunqcount;
 152
 153         /* temporary mess to retain structural compatibilty for now */
 154         union {
 155                 struct {
 156                         struct {
 157                                 int     gd_exisarmed;
 158                         } __cachealign;
 159                 };
 160                 long    gd_reserved02B[200];    /* used to be struct thread */
 161         };
 162         struct thread   gd_idlethread;
 163         SLGlobalData    gd_slab;                /* slab allocator */
 164         KMGlobalData    gd_kmslab;              /* kmalloc slab cache */
 165         int             gd_trap_nesting_level;  /* track traps */
 166         int             gd_vme_avail;           /* vm_map_entry reservation */
 167         struct vm_map_entry *gd_vme_base;       /* vm_map_entry reservation */
 168         struct systimerq gd_systimerq;          /* per-cpu system timers */
 169         int             gd_syst_nest;
 170         struct systimer gd_hardclock;           /* scheduler periodic */
 171         struct systimer gd_statclock;           /* statistics periodic */
 172         struct systimer gd_schedclock;          /* scheduler periodic */
 173         volatile __uint32_t gd_time_seconds;    /* uptime in seconds */
 174         volatile sysclock_t gd_cpuclock_base;   /* cpuclock relative base */
 175
 176         struct pipe     *gd_pipeq;              /* cache pipe structures */
 177         struct nchstats *gd_nchstats;           /* namecache effectiveness */
 178         int             gd_pipeqcount;          /* number of structures */
 179         sysid_t         gd_sysid_alloc;         /* allocate unique sysid */
 180
 181         struct tslpque  *gd_tsleep_hash;        /* tsleep/wakeup support */
 182         long            gd_processing_ipiq;
 183         int             gd_spinlocks;           /* Exclusive spinlocks held */
 184         struct systimer *gd_systimer_inprog;    /* in-progress systimer */
 185         int             gd_timer_running;
 186         u_int           gd_idle_repeat;         /* repeated switches to idle */
 187         int             gd_quick_color;         /* page-coloring helper */
 188         int             gd_cachedvnodes;        /* accum across all cpus */
 189         int             gd_rand_incr;           /* random pcpu incrementor */
 190         int             gd_activevnodes;        /* accum across all cpus */
 191         int             gd_inactivevnodes;      /* accum across all cpus */
 192         int             gd_ireserved[2];
 193         const char      *gd_infomsg;            /* debugging */
 194         struct lwkt_tokref gd_handoff;          /* hand-off tokref */
 195         void            *gd_delayed_wakeup[2];
 196         void            *gd_sample_pc;          /* sample program ctr/tr */
 197         uint64_t        gd_anoninum;            /* anonymous inode (pipes) */
 198         uint64_t        gd_forkid;              /* per-cpu unique inc ncpus */
 199         void            *gd_sample_sp;          /* sample stack pointer */
 200         uint64_t        gd_cpumask_simple;
 201         uint64_t        gd_cpumask_offset;
 202         struct vmstats  gd_vmstats;             /* pcpu local copy of vmstats */
 203         struct vmstats  gd_vmstats_adj;         /* pcpu adj for vmstats */
 204         struct callout  gd_loadav_callout;      /* loadavg calc */
 205         struct callout  gd_schedcpu_callout;    /* scheduler/stats */
 206         indefinite_info_t gd_indefinite;        /* scheduler cpu-bound */
 207         uint32_t        gd_loadav_nrunnable;    /* pcpu lwps nrunnable */
 208         uint32_t        gd_reserved32[1];
 209         struct lock     gd_sysctllock;          /* sysctl topology lock */
 210         uintptr_t       gd_debug1;
 211         uintptr_t       gd_debug2;
 212         long            gd_exislockcnt;
 213         void            *gd_preserved[1];       /* future fields */
 214         /* extended by <machine/globaldata.h> */
 215 };
 216
 217 typedef struct globaldata *globaldata_t;
 218
 219 #define RQB_IPIQ                0       /* 0001 */
 220 #define RQB_INTPEND             1       /* 0002 */
 221 #define RQB_AST_OWEUPC          2       /* 0004 */
 222 #define RQB_AST_SIGNAL          3       /* 0008 */
 223 #define RQB_AST_USER_RESCHED    4       /* 0010 */
 224 #define RQB_AST_LWKT_RESCHED    5       /* 0020 */
 225 #define RQB_UNUSED6             6       /* 0040 */
 226 #define RQB_TIMER               7       /* 0080 */
 227 #define RQB_RUNNING             8       /* 0100 */
 228 #define RQB_SPINNING            9       /* 0200 */
 229 #define RQB_QUICKRET            10      /* 0400 */
 230 #define RQB_KQUEUE              11      /* 0800 (only used by vkernel) */
 231 #define RQB_XINVLTLB            12      /* 1000 (HVM interlock) */
 232
 233 #define RQF_IPIQ                (1 << RQB_IPIQ)
 234 #define RQF_INTPEND             (1 << RQB_INTPEND)
 235 #define RQF_TIMER               (1 << RQB_TIMER)
 236 #define RQF_AST_OWEUPC          (1 << RQB_AST_OWEUPC)
 237 #define RQF_AST_SIGNAL          (1 << RQB_AST_SIGNAL)
 238 #define RQF_AST_USER_RESCHED    (1 << RQB_AST_USER_RESCHED)
 239 #define RQF_AST_LWKT_RESCHED    (1 << RQB_AST_LWKT_RESCHED)
 240 #define RQF_RUNNING             (1 << RQB_RUNNING)
 241 #define RQF_SPINNING            (1 << RQB_SPINNING)
 242 #define RQF_QUICKRET            (1 << RQB_QUICKRET)
 243 #define RQF_KQUEUE              (1 << RQB_KQUEUE)
 244 #define RQF_XINVLTLB            (1 << RQB_XINVLTLB)
 245
 246 #define RQF_AST_MASK            (RQF_AST_OWEUPC|RQF_AST_SIGNAL|\
 247                                  RQF_AST_USER_RESCHED|RQF_AST_LWKT_RESCHED)
 248 #define RQF_IDLECHECK_MASK      (RQF_IPIQ|RQF_INTPEND|RQF_TIMER|RQF_KQUEUE)
 249 #define RQF_IDLECHECK_WK_MASK   (RQF_IDLECHECK_MASK|RQF_AST_LWKT_RESCHED)
 250 #define RQF_SCHED_MASK          (RQF_IDLECHECK_MASK|RQF_AST_USER_RESCHED|\
 251                                  RQF_AST_LWKT_RESCHED)
 252 #define RQF_HVM_MASK            (RQF_IDLECHECK_MASK|RQF_AST_MASK|RQF_XINVLTLB)
 253
 254 /*
 255  * globaldata flags
 256  */
 257 #define GDF_KPRINTF             0x0001  /* kprintf() reentrancy */
 258 #define GDF_VIRTUSER            0x0002  /* used by vmm & vkernel */
 259
 260 #endif
 261
 262 /*
 263  * MANUAL DEBUG CODE FOR DEBUGGING LOCKUPS
 264  */
 265 #ifdef _KERNEL
 266
 267 #if 0
 268
 269 #define DEBUG_PUSH_INFO(msg)                            \
 270         const char *save_infomsg;                       \
 271         save_infomsg = mycpu->gd_infomsg;               \
 272         mycpu->gd_infomsg = msg                         \
 273
 274 #define DEBUG_POP_INFO()        mycpu->gd_infomsg = save_infomsg
 275
 276 #else
 277
 278 #define DEBUG_PUSH_INFO(msg)
 279 #define DEBUG_POP_INFO()
 280
 281 #endif
 282
 283 #endif
 284
 285 #ifdef _KERNEL
 286 struct globaldata *globaldata_find(int cpu);
 287 int is_globaldata_space(vm_offset_t saddr, vm_offset_t eaddr);
 288 #endif
 289
 290 #endif