kill tsol ("Trusted Solaris") aka TX ("Trusted Extensions")

[unleashed.git] / usr / src / cmd / zonestat / zonestatd / zonestatd.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2014 Garrett D'Amore <garrett@damore.org>
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */
#include <alloca.h>
#include <assert.h>
#include <dirent.h>
#include <dlfcn.h>
#include <door.h>
#include <errno.h>
#include <exacct.h>
#include <ctype.h>
#include <fcntl.h>
#include <kstat.h>
#include <libcontract.h>
#include <libintl.h>
#include <libscf.h>
#include <zonestat.h>
#include <zonestat_impl.h>
#include <limits.h>
#include <pool.h>
#include <procfs.h>
#include <priv.h>
#include <rctl.h>
#include <thread.h>
#include <signal.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
#include <stdlib.h>
#include <strings.h>
#include <synch.h>
#include <sys/acctctl.h>
#include <sys/contract/process.h>
#include <sys/ctfs.h>
#include <sys/fork.h>
#include <sys/param.h>
#include <sys/priocntl.h>
#include <sys/fxpriocntl.h>
#include <sys/processor.h>
#include <sys/pset.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/statvfs.h>
#include <sys/swap.h>
#include <sys/systeminfo.h>
#include <thread.h>
#include <sys/list.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/vm_usage.h>
#include <sys/wait.h>
#include <sys/zone.h>
#include <time.h>
#include <ucred.h>
#include <unistd.h>
#include <vm/anon.h>
#include <zone.h>
#include <zonestat.h>

#define MAX_PSET_NAME   1024    /* Taken from PV_NAME_MAX_LEN */
#define ZSD_PSET_UNLIMITED      UINT16_MAX
#define ZONESTAT_EXACCT_FILE    "/var/adm/exacct/zonestat-process"

/*
 * zonestatd implements gathering cpu and memory utilization data for
 * running zones.  It has these components:
 *
 * zsd_server:
 *      Door server to respond to client connections.  Each client
 *      will connect using libzonestat.so, which will open and
 *      call /var/tmp/.zonestat_door.  Each connecting client is given
 *      a file descriptor to the stat server.
 *
 *      The zsd_server also responds to zoneadmd, which reports when a
 *      new zone is booted.  This is used to fattach the zsd_server door
 *      into the new zone.
 *
 * zsd_stat_server:
 *      Receives client requests for the current utilization data.  Each
 *      client request will cause zonestatd to update the current utilization
 *      data by kicking the stat_thread.
 *
 *      If the client is in a non-global zone, the utilization data will
 *      be filtered to only show the given zone.  The usage by all other zones
 *      will be added to the system utilization.
 *
 * stat_thread:
 *      The stat thread implements querying the system to determine the
 *      current utilization data for each running zone.  This includes
 *      inspecting the system's processor set configuration, as well as details
 *      of each zone, such as their configured limits, and which processor
 *      sets they are running in.
 *
 *      The stat_thread will only update memory utilization data as often as
 *      the configured config/sample_interval on the zones-monitoring service.
 */

/*
 * The private vmusage structure unfortunately uses size_t types, and assumes
 * the caller's bitness matches the kernel's bitness.  Since the getvmusage()
 * system call is contracted, and zonestatd is 32 bit, the following structures
 * are used to interact with a 32bit or 64 bit kernel.
 */
typedef struct zsd_vmusage32 {
        id_t vmu_zoneid;
        uint_t vmu_type;
        id_t vmu_id;

        uint32_t vmu_rss_all;
        uint32_t vmu_rss_private;
        uint32_t vmu_rss_shared;
        uint32_t vmu_swap_all;
        uint32_t vmu_swap_private;
        uint32_t vmu_swap_shared;
} zsd_vmusage32_t;

typedef struct zsd_vmusage64 {
        id_t vmu_zoneid;
        uint_t vmu_type;
        id_t vmu_id;
        /*
         * An amd64 kernel will align the following uint64_t members, but a
         * 32bit i386 process will not without help.
         */
        int vmu_align_next_members_on_8_bytes;
        uint64_t vmu_rss_all;
        uint64_t vmu_rss_private;
        uint64_t vmu_rss_shared;
        uint64_t vmu_swap_all;
        uint64_t vmu_swap_private;
        uint64_t vmu_swap_shared;
} zsd_vmusage64_t;

struct zsd_zone;

/* Used to store a zone's usage of a pset */
typedef struct zsd_pset_usage {
        struct zsd_zone *zsu_zone;
        struct zsd_pset *zsu_pset;

        list_node_t     zsu_next;

        zoneid_t        zsu_zoneid;
        boolean_t       zsu_found;      /* zone bound at end of interval */
        boolean_t       zsu_active;     /* zone was bound during interval */
        boolean_t       zsu_new;        /* zone newly bound in this interval */
        boolean_t       zsu_deleted;    /* zone was unbound in this interval */
        boolean_t       zsu_empty;      /* no procs in pset in this interval */
        time_t          zsu_start;      /* time when zone was found in pset */
        hrtime_t        zsu_hrstart;    /* time when zone  was found in pset */
        uint64_t        zsu_cpu_shares;
        uint_t          zsu_scheds;     /* schedulers found in this pass */
        timestruc_t     zsu_cpu_usage;  /* cpu time used */
} zsd_pset_usage_t;

/* Used to store a pset's utilization */
typedef struct zsd_pset {
        psetid_t        zsp_id;
        list_node_t     zsp_next;
        char            zsp_name[ZS_PSETNAME_MAX];

        uint_t          zsp_cputype;    /* default, dedicated or shared */
        boolean_t       zsp_found;      /* pset found at end of interval */
        boolean_t       zsp_new;        /* pset new in this interval */
        boolean_t       zsp_deleted;    /* pset deleted in this interval */
        boolean_t       zsp_active;     /* pset existed during interval */
        boolean_t       zsp_empty;      /* no processes in pset */
        time_t          zsp_start;
        hrtime_t        zsp_hrstart;

        uint64_t        zsp_online;     /* online cpus in interval */
        uint64_t        zsp_size;       /* size in this interval */
        uint64_t        zsp_min;        /* configured min in this interval */
        uint64_t        zsp_max;        /* configured max in this interval */
        int64_t         zsp_importance; /* configured max in this interval */

        uint_t          zsp_scheds;     /* scheds of processes found in pset */
        uint64_t        zsp_cpu_shares; /* total shares in this interval */

        timestruc_t     zsp_total_time;
        timestruc_t     zsp_usage_kern;
        timestruc_t     zsp_usage_zones;

        /* Individual zone usages of pset */
        list_t          zsp_usage_list;
        int             zsp_nusage;

        /* Summed kstat values from individual cpus in pset */
        timestruc_t     zsp_idle;
        timestruc_t     zsp_intr;
        timestruc_t     zsp_kern;
        timestruc_t     zsp_user;

} zsd_pset_t;

/* Used to track an individual cpu's utilization as reported by kstats */
typedef struct zsd_cpu {
        processorid_t   zsc_id;
        list_node_t     zsc_next;
        psetid_t        zsc_psetid;
        psetid_t        zsc_psetid_prev;
        zsd_pset_t      *zsc_pset;

        boolean_t       zsc_found;      /* cpu online in this interval */
        boolean_t       zsc_onlined;    /* cpu onlined during this interval */
        boolean_t       zsc_offlined;   /* cpu offlined during this interval */
        boolean_t       zsc_active;     /* cpu online during this interval */
        boolean_t       zsc_allocated;  /* True if cpu has ever been found */

        /* kstats this interval */
        uint64_t        zsc_nsec_idle;
        uint64_t        zsc_nsec_intr;
        uint64_t        zsc_nsec_kern;
        uint64_t        zsc_nsec_user;

        /* kstats in most recent interval */
        uint64_t        zsc_nsec_idle_prev;
        uint64_t        zsc_nsec_intr_prev;
        uint64_t        zsc_nsec_kern_prev;
        uint64_t        zsc_nsec_user_prev;

        /* Total kstat increases since zonestatd started reading kstats */
        timestruc_t     zsc_idle;
        timestruc_t     zsc_intr;
        timestruc_t     zsc_kern;
        timestruc_t     zsc_user;

} zsd_cpu_t;

/* Used to describe an individual zone and its utilization */
typedef struct zsd_zone {
        zoneid_t        zsz_id;
        list_node_t     zsz_next;
        char            zsz_name[ZS_ZONENAME_MAX];
        uint_t          zsz_cputype;
        uint_t          zsz_iptype;
        time_t          zsz_start;
        hrtime_t        zsz_hrstart;

        char            zsz_pool[ZS_POOLNAME_MAX];
        char            zsz_pset[ZS_PSETNAME_MAX];
        int             zsz_default_sched;
        /* These are deduced by inspecting processes */
        psetid_t        zsz_psetid;
        uint_t          zsz_scheds;

        boolean_t       zsz_new;        /* zone booted during this interval */
        boolean_t       zsz_deleted;    /* halted during this interval */
        boolean_t       zsz_active;     /* running in this interval */
        boolean_t       zsz_empty;      /* no processes in this interval */
        boolean_t       zsz_gone;       /* not installed in this interval */
        boolean_t       zsz_found;      /* Running at end of this interval */

        uint64_t        zsz_cpu_shares;
        uint64_t        zsz_cpu_cap;
        uint64_t        zsz_ram_cap;
        uint64_t        zsz_locked_cap;
        uint64_t        zsz_vm_cap;

        uint64_t        zsz_cpus_online;
        timestruc_t     zsz_cpu_usage;  /* cpu time of cpu cap */
        timestruc_t     zsz_cap_time;   /* cpu time of cpu cap */
        timestruc_t     zsz_share_time; /* cpu time of share of cpu */
        timestruc_t     zsz_pset_time;  /* time of all psets zone is bound to */

        uint64_t        zsz_usage_ram;
        uint64_t        zsz_usage_locked;
        uint64_t        zsz_usage_vm;

        uint64_t        zsz_processes_cap;
        uint64_t        zsz_lwps_cap;
        uint64_t        zsz_shm_cap;
        uint64_t        zsz_shmids_cap;
        uint64_t        zsz_semids_cap;
        uint64_t        zsz_msgids_cap;
        uint64_t        zsz_lofi_cap;

        uint64_t        zsz_processes;
        uint64_t        zsz_lwps;
        uint64_t        zsz_shm;
        uint64_t        zsz_shmids;
        uint64_t        zsz_semids;
        uint64_t        zsz_msgids;
        uint64_t        zsz_lofi;

} zsd_zone_t;

/*
 * Used to track the cpu usage of an individual processes.
 *
 * zonestatd sweeps /proc each interval and charges the cpu usage of processes.
 * to their zone.  As processes exit, their extended accounting records are
 * read and the difference of their total and known usage is charged to their
 * zone.
 *
 * If a process is never seen in /proc, the total usage on its extended
 * accounting record will be charged to its zone.
 */
typedef struct zsd_proc {
        list_node_t     zspr_next;
        pid_t           zspr_ppid;
        psetid_t        zspr_psetid;
        zoneid_t        zspr_zoneid;
        int             zspr_sched;
        timestruc_t     zspr_usage;
} zsd_proc_t;

/* Used to track the overall resource usage of the system */
typedef struct zsd_system {

        uint64_t zss_ram_total;
        uint64_t zss_ram_kern;
        uint64_t zss_ram_zones;

        uint64_t zss_locked_kern;
        uint64_t zss_locked_zones;

        uint64_t zss_vm_total;
        uint64_t zss_vm_kern;
        uint64_t zss_vm_zones;

        uint64_t zss_swap_total;
        uint64_t zss_swap_used;

        timestruc_t zss_idle;
        timestruc_t zss_intr;
        timestruc_t zss_kern;
        timestruc_t zss_user;

        timestruc_t zss_cpu_total_time;
        timestruc_t zss_cpu_usage_kern;
        timestruc_t zss_cpu_usage_zones;

        uint64_t zss_maxpid;
        uint64_t zss_processes_max;
        uint64_t zss_lwps_max;
        uint64_t zss_shm_max;
        uint64_t zss_shmids_max;
        uint64_t zss_semids_max;
        uint64_t zss_msgids_max;
        uint64_t zss_lofi_max;

        uint64_t zss_processes;
        uint64_t zss_lwps;
        uint64_t zss_shm;
        uint64_t zss_shmids;
        uint64_t zss_semids;
        uint64_t zss_msgids;
        uint64_t zss_lofi;

        uint64_t zss_ncpus;
        uint64_t zss_ncpus_online;

} zsd_system_t;

/*
 * A dumping ground for various information and structures used to compute
 * utilization.
 *
 * This structure is used to track the system while clients are connected.
 * When The first client connects, a zsd_ctl is allocated and configured by
 * zsd_open().  When all clients disconnect, the zsd_ctl is closed.
 */
typedef struct zsd_ctl {
        kstat_ctl_t     *zsctl_kstat_ctl;

        /* To track extended accounting */
        int             zsctl_proc_fd;          /* Log currently being used */
        ea_file_t       zsctl_proc_eaf;
        struct stat64   zsctl_proc_stat;
        int             zsctl_proc_open;
        int             zsctl_proc_fd_next;     /* Log file to use next */
        ea_file_t       zsctl_proc_eaf_next;
        struct stat64   zsctl_proc_stat_next;
        int             zsctl_proc_open_next;

        /* pool configuration handle */
        pool_conf_t     *zsctl_pool_conf;
        int             zsctl_pool_status;
        int             zsctl_pool_changed;

        /* The above usage tacking structures */
        zsd_system_t    *zsctl_system;
        list_t          zsctl_zones;
        list_t          zsctl_psets;
        list_t          zsctl_cpus;
        zsd_cpu_t       *zsctl_cpu_array;
        zsd_proc_t      *zsctl_proc_array;

        /* Various system info */
        uint64_t        zsctl_maxcpuid;
        uint64_t        zsctl_maxproc;
        uint64_t        zsctl_kern_bits;
        uint64_t        zsctl_pagesize;

        /* Used to track time available under a cpu cap. */
        uint64_t        zsctl_hrtime;
        uint64_t        zsctl_hrtime_prev;
        timestruc_t     zsctl_hrtime_total;

        struct timeval  zsctl_timeofday;

        /* Caches for arrays allocated for use by various system calls */
        psetid_t        *zsctl_pset_cache;
        uint_t          zsctl_pset_ncache;
        processorid_t   *zsctl_cpu_cache;
        uint_t          zsctl_cpu_ncache;
        zoneid_t        *zsctl_zone_cache;
        uint_t          zsctl_zone_ncache;
        struct swaptable *zsctl_swap_cache;
        uint64_t        zsctl_swap_cache_size;
        uint64_t        zsctl_swap_cache_num;
        zsd_vmusage64_t *zsctl_vmusage_cache;
        uint64_t        zsctl_vmusage_cache_num;

        /* Info about procfs for scanning /proc */
        struct dirent   *zsctl_procfs_dent;
        long            zsctl_procfs_dent_size;
        pool_value_t    *zsctl_pool_vals[3];

        /* Counts on tracked entities */
        uint_t          zsctl_nzones;
        uint_t          zsctl_npsets;
        uint_t          zsctl_npset_usages;
} zsd_ctl_t;

zsd_ctl_t               *g_ctl;
boolean_t               g_open;         /* True if g_ctl is open */
int                     g_hasclient;    /* True if any clients are connected */

/*
 * The usage cache is updated by the stat_thread, and copied to clients by
 * the zsd_stat_server.  Mutex and cond are to synchronize between the
 * stat_thread and the stat_server.
 */
zs_usage_cache_t        *g_usage_cache;
mutex_t                 g_usage_cache_lock;
cond_t                  g_usage_cache_kick;
uint_t                  g_usage_cache_kickers;
cond_t                  g_usage_cache_wait;
char                    *g_usage_cache_buf;
uint_t                  g_usage_cache_bufsz;
uint64_t                g_gen_next;

/* fds of door servers */
int                     g_server_door;
int                     g_stat_door;

/*
 * Starting and current time.  Used to throttle memory calculation, and to
 * mark new zones and psets with their boot and creation time.
 */
time_t                  g_now;
time_t                  g_start;
hrtime_t                g_hrnow;
hrtime_t                g_hrstart;
uint64_t                g_interval;

/*
 * main() thread.
 */
thread_t                g_main;

/* PRINTFLIKE1 */
static void
zsd_warn(const char *fmt, ...)
{
        va_list alist;

        va_start(alist, fmt);

        (void) fprintf(stderr, gettext("zonestat: Warning: "));
        (void) vfprintf(stderr, fmt, alist);
        (void) fprintf(stderr, "\n");
        va_end(alist);
}

/* PRINTFLIKE1 */
static void
zsd_error(const char *fmt, ...)
{
        va_list alist;

        va_start(alist, fmt);

        (void) fprintf(stderr, gettext("zonestat: Error: "));
        (void) vfprintf(stderr, fmt, alist);
        (void) fprintf(stderr, "\n");
        va_end(alist);
        exit(1);
}

/* Turns on extended accounting if not configured externally */
int
zsd_enable_cpu_stats()
{
        char *path = ZONESTAT_EXACCT_FILE;
        char oldfile[MAXPATHLEN];
        int ret, state = AC_ON;
        ac_res_t res[6];

        /*
         * Start a new accounting file  if accounting not configured
         * externally.
         */

        res[0].ar_id = AC_PROC_PID;
        res[0].ar_state = AC_ON;
        res[1].ar_id = AC_PROC_ANCPID;
        res[1].ar_state = AC_ON;
        res[2].ar_id = AC_PROC_CPU;
        res[2].ar_state = AC_ON;
        res[3].ar_id = AC_PROC_TIME;
        res[3].ar_state = AC_ON;
        res[4].ar_id = AC_PROC_ZONENAME;
        res[4].ar_state = AC_ON;
        res[5].ar_id = AC_NONE;
        res[5].ar_state = AC_ON;
        if (acctctl(AC_PROC | AC_RES_SET, res, sizeof (res)) != 0) {
                zsd_warn(gettext("Unable to set accounting resources"));
                return (-1);
        }
        /* Only set accounting file if none is configured */
        ret = acctctl(AC_PROC | AC_FILE_GET, oldfile, sizeof (oldfile));
        if (ret < 0) {

                (void) unlink(path);
                if (acctctl(AC_PROC | AC_FILE_SET, path, strlen(path) + 1)
                    == -1) {
                        zsd_warn(gettext("Unable to set accounting file"));
                        return (-1);
                }
        }
        if (acctctl(AC_PROC | AC_STATE_SET, &state, sizeof (state)) == -1) {
                zsd_warn(gettext("Unable to enable accounting"));
                return (-1);
        }
        return (0);
}

/* Turns off extended accounting if not configured externally */
int
zsd_disable_cpu_stats()
{
        char *path = ZONESTAT_EXACCT_FILE;
        int ret, state = AC_OFF;
        ac_res_t res[6];
        char oldfile[MAXPATHLEN];

        /* If accounting file is externally configured, leave it alone */
        ret = acctctl(AC_PROC | AC_FILE_GET, oldfile, sizeof (oldfile));
        if (ret == 0 && strcmp(oldfile, path) != 0)
                return (0);

        res[0].ar_id = AC_PROC_PID;
        res[0].ar_state = AC_OFF;
        res[1].ar_id = AC_PROC_ANCPID;
        res[1].ar_state = AC_OFF;
        res[2].ar_id = AC_PROC_CPU;
        res[2].ar_state = AC_OFF;
        res[3].ar_id = AC_PROC_TIME;
        res[3].ar_state = AC_OFF;
        res[4].ar_id = AC_PROC_ZONENAME;
        res[4].ar_state = AC_OFF;
        res[5].ar_id = AC_NONE;
        res[5].ar_state = AC_OFF;
        if (acctctl(AC_PROC | AC_RES_SET, res, sizeof (res)) != 0) {
                zsd_warn(gettext("Unable to clear accounting resources"));
                return (-1);
        }
        if (acctctl(AC_PROC | AC_FILE_SET, NULL, 0) == -1) {
                zsd_warn(gettext("Unable to clear accounting file"));
                return (-1);
        }
        if (acctctl(AC_PROC | AC_STATE_SET, &state, sizeof (state)) == -1) {
                zsd_warn(gettext("Unable to diable accounting"));
                return (-1);
        }

        (void) unlink(path);
        return (0);
}

/*
 * If not configured externally, deletes the current extended accounting file
 * and starts a new one.
 *
 * Since the stat_thread holds an open handle to the accounting file, it will
 * read all remaining entries from the old file before switching to
 * read the new one.
 */
int
zsd_roll_exacct(void)
{
        int ret;
        char *path = ZONESTAT_EXACCT_FILE;
        char oldfile[MAXPATHLEN];

        /* If accounting file is externally configured, leave it alone */
        ret = acctctl(AC_PROC | AC_FILE_GET, oldfile, sizeof (oldfile));
        if (ret == 0 && strcmp(oldfile, path) != 0)
                return (0);

        if (unlink(path) != 0)
                /* Roll it next time */
                return (0);

        if (acctctl(AC_PROC | AC_FILE_SET, path, strlen(path) + 1) == -1) {
                zsd_warn(gettext("Unable to set accounting file"));
                return (-1);
        }
        return (0);
}

/* Contract stuff for zone_enter() */
int
init_template(void)
{
        int fd;
        int err = 0;

        fd = open64(CTFS_ROOT "/process/template", O_RDWR);
        if (fd == -1)
                return (-1);

        /*
         * For now, zoneadmd doesn't do anything with the contract.
         * Deliver no events, don't inherit, and allow it to be orphaned.
         */
        err |= ct_tmpl_set_critical(fd, 0);
        err |= ct_tmpl_set_informative(fd, 0);
        err |= ct_pr_tmpl_set_fatal(fd, CT_PR_EV_HWERR);
        err |= ct_pr_tmpl_set_param(fd, CT_PR_PGRPONLY | CT_PR_REGENT);
        if (err || ct_tmpl_activate(fd)) {
                (void) close(fd);
                return (-1);
        }

        return (fd);
}

/*
 * Contract stuff for zone_enter()
 */
int
contract_latest(ctid_t *id)
{
        int cfd, r;
        ct_stathdl_t st;
        ctid_t result;

        if ((cfd = open64(CTFS_ROOT "/process/latest", O_RDONLY)) == -1)
                return (errno);

        if ((r = ct_status_read(cfd, CTD_COMMON, &st)) != 0) {
                (void) close(cfd);
                return (r);
        }

        result = ct_status_get_id(st);
        ct_status_free(st);
        (void) close(cfd);

        *id = result;
        return (0);
}

static int
close_on_exec(int fd)
{
        int flags = fcntl(fd, F_GETFD, 0);
        if ((flags != -1) && (fcntl(fd, F_SETFD, flags | FD_CLOEXEC) != -1))
                return (0);
        return (-1);
}

int
contract_open(ctid_t ctid, const char *type, const char *file, int oflag)
{
        char path[PATH_MAX];
        int n, fd;

        if (type == NULL)
                type = "all";

        n = snprintf(path, PATH_MAX, CTFS_ROOT "/%s/%ld/%s", type, ctid, file);
        if (n >= sizeof (path)) {
                errno = ENAMETOOLONG;
                return (-1);
        }

        fd = open64(path, oflag);
        if (fd != -1) {
                if (close_on_exec(fd) == -1) {
                        int err = errno;
                        (void) close(fd);
                        errno = err;
                        return (-1);
                }
        }
        return (fd);
}

int
contract_abandon_id(ctid_t ctid)
{
        int fd, err;

        fd = contract_open(ctid, "all", "ctl", O_WRONLY);
        if (fd == -1)
                return (errno);

        err = ct_ctl_abandon(fd);
        (void) close(fd);

        return (err);
}
/*
 * Attach the zsd_server to a zone.  Called for each zone when zonestatd
 * starts, and for each newly booted zone when zoneadmd contacts the zsd_server
 *
 * Zone_enter is used to avoid reaching into zone to fattach door.
 */
static void
zsd_fattach_zone(zoneid_t zid, int door, boolean_t detach_only)
{
        char *path = ZS_DOOR_PATH;
        int fd, pid, stat, tmpl_fd;
        ctid_t ct;

        if ((tmpl_fd = init_template()) == -1) {
                zsd_warn("Unable to init template");
                return;
        }

        pid = forkx(0);
        if (pid < 0) {
                (void) ct_tmpl_clear(tmpl_fd);
                zsd_warn(gettext(
                    "Unable to fork to add zonestat to zoneid %d\n"), zid);
                return;
        }

        if (pid == 0) {
                (void) ct_tmpl_clear(tmpl_fd);
                (void) close(tmpl_fd);
                if (zid != 0 && zone_enter(zid) != 0) {
                        if (errno == EINVAL) {
                                _exit(0);
                        }
                        _exit(1);
                }
                (void) fdetach(path);
                (void) unlink(path);
                if (detach_only)
                        _exit(0);
                fd = open(path, O_CREAT|O_RDWR, 0644);
                if (fd < 0)
                        _exit(2);
                if (fattach(door, path) != 0)
                        _exit(3);
                _exit(0);
        }
        if (contract_latest(&ct) == -1)
                ct = -1;
        (void) ct_tmpl_clear(tmpl_fd);
        (void) close(tmpl_fd);
        (void) contract_abandon_id(ct);
        while (waitpid(pid, &stat, 0) != pid)
                ;
        if (WIFEXITED(stat) && WEXITSTATUS(stat) == 0)
                return;

        zsd_warn(gettext("Unable to attach door to zoneid: %d"), zid);

        if (WEXITSTATUS(stat) == 1)
                zsd_warn(gettext("Cannot entering zone"));
        else if (WEXITSTATUS(stat) == 2)
                zsd_warn(gettext("Unable to create door file: %s"), path);
        else if (WEXITSTATUS(stat) == 3)
                zsd_warn(gettext("Unable to fattach file: %s"), path);

        zsd_warn(gettext("Internal error entering zone: %d"), zid);
}

/*
 * Zone lookup and allocation functions to manage list of currently running
 * zones.
 */
static zsd_zone_t *
zsd_lookup_zone(zsd_ctl_t *ctl, char *zonename, zoneid_t zoneid)
{
        zsd_zone_t *zone;

        for (zone = list_head(&ctl->zsctl_zones); zone != NULL;
            zone = list_next(&ctl->zsctl_zones, zone)) {
                if (strcmp(zone->zsz_name, zonename) == 0) {
                        if (zoneid != -1)
                                zone->zsz_id = zoneid;
                        return (zone);
                }
        }
        return (NULL);
}

static zsd_zone_t *
zsd_lookup_zone_byid(zsd_ctl_t *ctl, zoneid_t zoneid)
{
        zsd_zone_t *zone;

        for (zone = list_head(&ctl->zsctl_zones); zone != NULL;
            zone = list_next(&ctl->zsctl_zones, zone)) {
                if (zone->zsz_id == zoneid)
                        return (zone);
        }
        return (NULL);
}

static zsd_zone_t *
zsd_allocate_zone(zsd_ctl_t *ctl, char *zonename, zoneid_t zoneid)
{
        zsd_zone_t *zone;

        if ((zone = (zsd_zone_t *)calloc(1, sizeof (zsd_zone_t))) == NULL)
                return (NULL);

        (void) strlcpy(zone->zsz_name, zonename, sizeof (zone->zsz_name));
        zone->zsz_id = zoneid;
        zone->zsz_found = B_FALSE;

        /*
         * Allocate as deleted so if not found in first pass, zone is deleted
         * from list.  This can happen if zone is returned by zone_list, but
         * exits before first attempt to fetch zone details.
         */
        zone->zsz_start = g_now;
        zone->zsz_hrstart = g_hrnow;
        zone->zsz_deleted = B_TRUE;

        zone->zsz_cpu_shares = ZS_LIMIT_NONE;
        zone->zsz_cpu_cap = ZS_LIMIT_NONE;
        zone->zsz_ram_cap = ZS_LIMIT_NONE;
        zone->zsz_locked_cap = ZS_LIMIT_NONE;
        zone->zsz_vm_cap = ZS_LIMIT_NONE;

        zone->zsz_processes_cap = ZS_LIMIT_NONE;
        zone->zsz_lwps_cap = ZS_LIMIT_NONE;
        zone->zsz_shm_cap = ZS_LIMIT_NONE;
        zone->zsz_shmids_cap = ZS_LIMIT_NONE;
        zone->zsz_semids_cap = ZS_LIMIT_NONE;
        zone->zsz_msgids_cap = ZS_LIMIT_NONE;
        zone->zsz_lofi_cap = ZS_LIMIT_NONE;

        ctl->zsctl_nzones++;

        return (zone);
}

static zsd_zone_t *
zsd_lookup_insert_zone(zsd_ctl_t *ctl, char *zonename, zoneid_t zoneid)
{
        zsd_zone_t *zone, *tmp;

        if ((zone = zsd_lookup_zone(ctl, zonename, zoneid)) != NULL)
                return (zone);

        if ((zone = zsd_allocate_zone(ctl, zonename, zoneid)) == NULL)
                return (NULL);

        /* Insert sorted by zonename */
        tmp = list_head(&ctl->zsctl_zones);
        while (tmp != NULL && strcmp(zonename, tmp->zsz_name) > 0)
                tmp = list_next(&ctl->zsctl_zones, tmp);

        list_insert_before(&ctl->zsctl_zones, tmp, zone);
        return (zone);
}

/*
 * Mark all zones as not existing.  As zones are found, they will
 * be marked as existing.  If a zone is not found, then it must have
 * halted.
 */
static void
zsd_mark_zones_start(zsd_ctl_t *ctl)
{

        zsd_zone_t *zone;

        for (zone = list_head(&ctl->zsctl_zones); zone != NULL;
            zone = list_next(&ctl->zsctl_zones, zone)) {
                zone->zsz_found = B_FALSE;
        }
}

/*
 * Mark each zone as not using pset.  If processes are found using the
 * pset, the zone will remain bound to the pset.  If none of a zones
 * processes are bound to the pset, the zone's usage of the pset will
 * be deleted.
 *
 */
static void
zsd_mark_pset_usage_start(zsd_pset_t *pset)
{
        zsd_pset_usage_t *usage;

        for (usage = list_head(&pset->zsp_usage_list);
            usage != NULL;
            usage = list_next(&pset->zsp_usage_list, usage)) {
                usage->zsu_found = B_FALSE;
                usage->zsu_empty = B_TRUE;
        }
}

/*
 * Mark each pset as not existing.  If a pset is found, it will be marked
 * as existing.  If a pset is not found, it wil be deleted.
 */
static void
zsd_mark_psets_start(zsd_ctl_t *ctl)
{
        zsd_pset_t *pset;

        for (pset = list_head(&ctl->zsctl_psets); pset != NULL;
            pset = list_next(&ctl->zsctl_psets, pset)) {
                pset->zsp_found = B_FALSE;
                zsd_mark_pset_usage_start(pset);
        }
}

/*
 * A pset was found.  Update its information
 */
static void
zsd_mark_pset_found(zsd_pset_t *pset, uint_t type, uint64_t online,
    uint64_t size, uint64_t min, uint64_t max, int64_t importance)
{
        pset->zsp_empty = B_TRUE;
        pset->zsp_deleted = B_FALSE;

        assert(pset->zsp_found == B_FALSE);

        /* update pset flags */
        if (pset->zsp_active == B_FALSE)
                /* pset not seen on previous interval.  It is new. */
                pset->zsp_new = B_TRUE;
        else
                pset->zsp_new = B_FALSE;

        pset->zsp_found = B_TRUE;
        pset->zsp_cputype = type;
        pset->zsp_online = online;
        pset->zsp_size = size;
        pset->zsp_min = min;
        pset->zsp_max = max;
        pset->zsp_importance = importance;
        pset->zsp_cpu_shares = 0;
        pset->zsp_scheds = 0;
        pset->zsp_active = B_TRUE;
}

/*
 * A zone's process was found using a pset. Charge the process to the pset and
 * the per-zone data for the pset.
 */
static void
zsd_mark_pset_usage_found(zsd_pset_usage_t *usage, uint_t sched)
{
        zsd_zone_t *zone = usage->zsu_zone;
        zsd_pset_t *pset = usage->zsu_pset;

        /* Nothing to do if already found */
        if (usage->zsu_found == B_TRUE)
                goto add_stats;

        usage->zsu_found = B_TRUE;
        usage->zsu_empty = B_FALSE;

        usage->zsu_deleted = B_FALSE;
        /* update usage flags */
        if (usage->zsu_active == B_FALSE)
                usage->zsu_new = B_TRUE;
        else
                usage->zsu_new = B_FALSE;

        usage->zsu_scheds = 0;
        usage->zsu_cpu_shares = ZS_LIMIT_NONE;
        usage->zsu_active = B_TRUE;
        pset->zsp_empty = B_FALSE;
        zone->zsz_empty = B_FALSE;

add_stats:
        /* Detect zone's pset id, and if it is bound to multiple psets */
        if (zone->zsz_psetid == ZS_PSET_ERROR)
                zone->zsz_psetid = pset->zsp_id;
        else if (zone->zsz_psetid != pset->zsp_id)
                zone->zsz_psetid = ZS_PSET_MULTI;

        usage->zsu_scheds |= sched;
        pset->zsp_scheds |= sched;
        zone->zsz_scheds |= sched;

        /* Record if FSS is co-habitating with conflicting scheduler */
        if ((pset->zsp_scheds & ZS_SCHED_FSS) &&
            usage->zsu_scheds & (
            ZS_SCHED_TS | ZS_SCHED_IA | ZS_SCHED_FX)) {
                usage->zsu_scheds |= ZS_SCHED_CONFLICT;

                pset->zsp_scheds |= ZS_SCHED_CONFLICT;
        }

}

/* Add cpu time for a process to a pset, zone, and system totals */
static void
zsd_add_usage(zsd_ctl_t *ctl, zsd_pset_usage_t *usage, timestruc_t *delta)
{
        zsd_system_t *system = ctl->zsctl_system;
        zsd_zone_t *zone = usage->zsu_zone;
        zsd_pset_t *pset = usage->zsu_pset;

        TIMESTRUC_ADD_TIMESTRUC(usage->zsu_cpu_usage, *delta);
        TIMESTRUC_ADD_TIMESTRUC(pset->zsp_usage_zones, *delta);
        TIMESTRUC_ADD_TIMESTRUC(zone->zsz_cpu_usage, *delta);
        TIMESTRUC_ADD_TIMESTRUC(system->zss_cpu_usage_zones, *delta);
}

/* Determine which processor sets have been deleted */
static void
zsd_mark_psets_end(zsd_ctl_t *ctl)
{
        zsd_pset_t *pset, *tmp;

        /*
         * Mark pset as not exists, and deleted if it existed
         * previous interval.
         */
        pset = list_head(&ctl->zsctl_psets);
        while (pset != NULL) {
                if (pset->zsp_found == B_FALSE) {
                        pset->zsp_empty = B_TRUE;
                        if (pset->zsp_deleted == B_TRUE) {
                                tmp = pset;
                                pset = list_next(&ctl->zsctl_psets, pset);
                                list_remove(&ctl->zsctl_psets, tmp);
                                free(tmp);
                                ctl->zsctl_npsets--;
                                continue;
                        } else {
                                /* Pset vanished during this interval */
                                pset->zsp_new = B_FALSE;
                                pset->zsp_deleted = B_TRUE;
                                pset->zsp_active = B_TRUE;
                        }
                }
                pset = list_next(&ctl->zsctl_psets, pset);
        }
}

/* Determine which zones are no longer bound to processor sets */
static void
zsd_mark_pset_usages_end(zsd_ctl_t *ctl)
{
        zsd_pset_t *pset;
        zsd_zone_t *zone;
        zsd_pset_usage_t *usage, *tmp;

        /*
         * Mark pset as not exists, and deleted if it existed previous
         * interval.
         */
        for (pset = list_head(&ctl->zsctl_psets); pset != NULL;
            pset = list_next(&ctl->zsctl_psets, pset)) {
                usage = list_head(&pset->zsp_usage_list);
                while (usage != NULL) {
                        /*
                         * Mark pset as not exists, and deleted if it existed
                         * previous interval.
                         */
                        if (usage->zsu_found == B_FALSE ||
                            usage->zsu_zone->zsz_deleted == B_TRUE ||
                            usage->zsu_pset->zsp_deleted == B_TRUE) {
                                tmp = usage;
                                usage = list_next(&pset->zsp_usage_list,
                                    usage);
                                list_remove(&pset->zsp_usage_list, tmp);
                                free(tmp);
                                pset->zsp_nusage--;
                                ctl->zsctl_npset_usages--;
                                continue;
                        } else {
                                usage->zsu_new = B_FALSE;
                                usage->zsu_deleted = B_TRUE;
                                usage->zsu_active = B_TRUE;
                        }
                        /* Add cpu shares for usages that are in FSS */
                        zone = usage->zsu_zone;
                        if (usage->zsu_scheds & ZS_SCHED_FSS &&
                            zone->zsz_cpu_shares != ZS_SHARES_UNLIMITED &&
                            zone->zsz_cpu_shares != 0) {
                                zone = usage->zsu_zone;
                                usage->zsu_cpu_shares = zone->zsz_cpu_shares;
                                pset->zsp_cpu_shares += zone->zsz_cpu_shares;
                        }
                        usage = list_next(&pset->zsp_usage_list,
                            usage);
                }
        }
}

/* A zone has been found.  Update its information */
static void
zsd_mark_zone_found(zsd_ctl_t *ctl, zsd_zone_t *zone, uint64_t cpu_shares,
    uint64_t cpu_cap, uint64_t ram_cap, uint64_t locked_cap,
    uint64_t vm_cap, uint64_t processes_cap, uint64_t processes,
    uint64_t lwps_cap, uint64_t lwps, uint64_t shm_cap, uint64_t shm,
    uint64_t shmids_cap, uint64_t shmids, uint64_t semids_cap,
    uint64_t semids, uint64_t msgids_cap, uint64_t msgids, uint64_t lofi_cap,
    uint64_t lofi, char *poolname, char *psetname, uint_t sched, uint_t cputype,
    uint_t iptype)
{
        zsd_system_t *sys = ctl->zsctl_system;

        assert(zone->zsz_found == B_FALSE);

        /*
         * Mark zone as exists, and new if it did not exist in previous
         * interval.
         */
        zone->zsz_found = B_TRUE;
        zone->zsz_empty = B_TRUE;
        zone->zsz_deleted = B_FALSE;

        /*
         * Zone is new.  Assume zone's properties are the same over entire
         * interval.
         */
        if (zone->zsz_active == B_FALSE)
                zone->zsz_new = B_TRUE;
        else
                zone->zsz_new = B_FALSE;

        (void) strlcpy(zone->zsz_pool, poolname, sizeof (zone->zsz_pool));
        (void) strlcpy(zone->zsz_pset, psetname, sizeof (zone->zsz_pset));
        zone->zsz_default_sched = sched;

        /* Schedulers updated later as processes are found */
        zone->zsz_scheds = 0;

        /* Cpus updated later as psets bound are identified */
        zone->zsz_cpus_online = 0;

        zone->zsz_cputype = cputype;
        zone->zsz_iptype = iptype;
        zone->zsz_psetid = ZS_PSET_ERROR;
        zone->zsz_cpu_cap = cpu_cap;
        zone->zsz_cpu_shares = cpu_shares;
        zone->zsz_ram_cap = ram_cap;
        zone->zsz_locked_cap = locked_cap;
        zone->zsz_vm_cap = vm_cap;
        zone->zsz_processes_cap = processes_cap;
        zone->zsz_processes = processes;
        zone->zsz_lwps_cap = lwps_cap;
        zone->zsz_lwps = lwps;
        zone->zsz_shm_cap = shm_cap;
        zone->zsz_shm = shm;
        zone->zsz_shmids_cap = shmids_cap;
        zone->zsz_shmids = shmids;
        zone->zsz_semids_cap = semids_cap;
        zone->zsz_semids = semids;
        zone->zsz_msgids_cap = msgids_cap;
        zone->zsz_msgids = msgids;
        zone->zsz_lofi_cap = lofi_cap;
        zone->zsz_lofi = lofi;

        sys->zss_processes += processes;
        sys->zss_lwps += lwps;
        sys->zss_shm += shm;
        sys->zss_shmids += shmids;
        sys->zss_semids += semids;
        sys->zss_msgids += msgids;
        sys->zss_lofi += lofi;
        zone->zsz_active = B_TRUE;
}


/* Determine which zones have halted */
static void
zsd_mark_zones_end(zsd_ctl_t *ctl)
{
        zsd_zone_t *zone, *tmp;

        /*
         * Mark zone as not existing, or delete if it did not exist in
         * previous interval.
         */
        zone = list_head(&ctl->zsctl_zones);
        while (zone != NULL) {
                if (zone->zsz_found == B_FALSE) {
                        zone->zsz_empty = B_TRUE;
                        if (zone->zsz_deleted == B_TRUE) {
                                /*
                                 * Zone deleted in prior interval,
                                 * so it no longer exists.
                                 */
                                tmp = zone;
                                zone = list_next(&ctl->zsctl_zones, zone);
                                list_remove(&ctl->zsctl_zones, tmp);
                                free(tmp);
                                ctl->zsctl_nzones--;
                                continue;
                        } else {
                                zone->zsz_new = B_FALSE;
                                zone->zsz_deleted = B_TRUE;
                                zone->zsz_active = B_TRUE;
                        }
                }
                zone = list_next(&ctl->zsctl_zones, zone);
        }
}

/*
 * Mark cpus as not existing.  If a cpu is found, it will be updated.  If
 * a cpu is not found, then it must have gone offline, so it will be
 * deleted.
 *
 * The kstat tracking data is rolled so that the usage since the previous
 * interval can be determined.
 */
static void
zsd_mark_cpus_start(zsd_ctl_t *ctl, boolean_t roll)
{
        zsd_cpu_t *cpu;

        /*
         * Mark all cpus as not existing.  As cpus are found, they will
         * be marked as existing.
         */
        for (cpu = list_head(&ctl->zsctl_cpus); cpu != NULL;
            cpu = list_next(&ctl->zsctl_cpus, cpu)) {
                cpu->zsc_found = B_FALSE;
                if (cpu->zsc_active == B_TRUE && roll) {
                        cpu->zsc_psetid_prev = cpu->zsc_psetid;
                        cpu->zsc_nsec_idle_prev = cpu->zsc_nsec_idle;
                        cpu->zsc_nsec_intr_prev = cpu->zsc_nsec_intr;
                        cpu->zsc_nsec_kern_prev = cpu->zsc_nsec_kern;
                        cpu->zsc_nsec_user_prev = cpu->zsc_nsec_user;
                }
        }
}

/*
 * An array the size of the maximum number of cpus is kept.  Within this array
 * a list of the online cpus is maintained.
 */
zsd_cpu_t *
zsd_lookup_insert_cpu(zsd_ctl_t *ctl, processorid_t cpuid)
{
        zsd_cpu_t *cpu;

        assert(cpuid < ctl->zsctl_maxcpuid);
        cpu = &(ctl->zsctl_cpu_array[cpuid]);
        assert(cpuid == cpu->zsc_id);

        if (cpu->zsc_allocated == B_FALSE) {
                cpu->zsc_allocated = B_TRUE;
                list_insert_tail(&ctl->zsctl_cpus, cpu);
        }
        return (cpu);
}

/* A cpu has been found.  Update its information */
static void
zsd_mark_cpu_found(zsd_cpu_t *cpu, zsd_pset_t *pset, psetid_t psetid)
{
        /*
         * legacy processor sets, the cpu may move while zonestatd is
         * inspecting, causing it to be found twice.  In this case, just
         * leave cpu in the first processor set in which it was found.
         */
        if (cpu->zsc_found == B_TRUE)
                return;

        /* Mark cpu as online */
        cpu->zsc_found = B_TRUE;
        cpu->zsc_offlined = B_FALSE;
        cpu->zsc_pset = pset;
        /*
         * cpu is newly online.
         */
        if (cpu->zsc_active == B_FALSE) {
                /*
                 * Cpu is newly online.
                 */
                cpu->zsc_onlined = B_TRUE;
                cpu->zsc_psetid = psetid;
                cpu->zsc_psetid_prev = psetid;
        } else {
                /*
                 * cpu online during previous interval.  Save properties at
                 * start of interval
                 */
                cpu->zsc_onlined = B_FALSE;
                cpu->zsc_psetid = psetid;

        }
        cpu->zsc_active = B_TRUE;
}

/* Remove all offlined cpus from the list of tracked cpus */
static void
zsd_mark_cpus_end(zsd_ctl_t *ctl)
{
        zsd_cpu_t *cpu, *tmp;
        int id;

        /* Mark cpu as online or offline */
        cpu = list_head(&ctl->zsctl_cpus);
        while (cpu != NULL) {
                if (cpu->zsc_found == B_FALSE) {
                        if (cpu->zsc_offlined == B_TRUE) {
                                /*
                                 * cpu offlined in prior interval. It is gone.
                                 */
                                tmp = cpu;
                                cpu = list_next(&ctl->zsctl_cpus, cpu);
                                list_remove(&ctl->zsctl_cpus, tmp);
                                /* Clear structure for future use */
                                id = tmp->zsc_id;
                                bzero(tmp, sizeof (zsd_cpu_t));
                                tmp->zsc_id = id;
                                tmp->zsc_allocated = B_FALSE;
                                tmp->zsc_psetid = ZS_PSET_ERROR;
                                tmp->zsc_psetid_prev = ZS_PSET_ERROR;

                        } else {
                                /*
                                 * cpu online at start of interval.  Treat
                                 * as still online, since it was online for
                                 * some portion of the interval.
                                 */
                                cpu->zsc_offlined = B_TRUE;
                                cpu->zsc_onlined = B_FALSE;
                                cpu->zsc_active = B_TRUE;
                                cpu->zsc_psetid = cpu->zsc_psetid_prev;
                                cpu->zsc_pset = NULL;
                        }
                }
                cpu = list_next(&ctl->zsctl_cpus, cpu);
        }
}

/* Some utility functions for managing the list of processor sets */
static zsd_pset_t *
zsd_lookup_pset_byid(zsd_ctl_t *ctl, psetid_t psetid)
{
        zsd_pset_t *pset;

        for (pset = list_head(&ctl->zsctl_psets); pset != NULL;
            pset = list_next(&ctl->zsctl_psets, pset)) {
                if (pset->zsp_id == psetid)
                        return (pset);
        }
        return (NULL);
}

static zsd_pset_t *
zsd_lookup_pset(zsd_ctl_t *ctl, char *psetname, psetid_t psetid)
{
        zsd_pset_t *pset;

        for (pset = list_head(&ctl->zsctl_psets); pset != NULL;
            pset = list_next(&ctl->zsctl_psets, pset)) {
                if (strcmp(pset->zsp_name, psetname) == 0) {
                        if (psetid != -1)
                                pset->zsp_id = psetid;
                        return (pset);
                }
        }
        return (NULL);
}

static zsd_pset_t *
zsd_allocate_pset(zsd_ctl_t *ctl, char *psetname, psetid_t psetid)
{
        zsd_pset_t *pset;

        if ((pset = (zsd_pset_t *)calloc(1, sizeof (zsd_pset_t))) == NULL)
                return (NULL);

        (void) strlcpy(pset->zsp_name, psetname, sizeof (pset->zsp_name));
        pset->zsp_id = psetid;
        pset->zsp_found = B_FALSE;
        /*
         * Allocate as deleted so if not found in first pass, pset is deleted
         * from list.  This can happen if pset is returned by pset_list, but
         * is destroyed before first attempt to fetch pset details.
         */
        list_create(&pset->zsp_usage_list, sizeof (zsd_pset_usage_t),
            offsetof(zsd_pset_usage_t, zsu_next));

        pset->zsp_hrstart = g_hrnow;
        pset->zsp_deleted = B_TRUE;
        pset->zsp_empty = B_TRUE;
        ctl->zsctl_npsets++;

        return (pset);
}

static zsd_pset_t *
zsd_lookup_insert_pset(zsd_ctl_t *ctl, char *psetname, psetid_t psetid)
{
        zsd_pset_t *pset, *tmp;

        if ((pset = zsd_lookup_pset(ctl, psetname, psetid)) != NULL)
                return (pset);

        if ((pset = zsd_allocate_pset(ctl, psetname, psetid)) == NULL)
                return (NULL);

        /* Insert sorted by psetname */
        tmp = list_head(&ctl->zsctl_psets);
        while (tmp != NULL && strcmp(psetname, tmp->zsp_name) > 0)
                tmp = list_next(&ctl->zsctl_psets, tmp);

        list_insert_before(&ctl->zsctl_psets, tmp, pset);
        return (pset);
}

/* Some utility functions for managing the list of zones using each pset */
static zsd_pset_usage_t *
zsd_lookup_usage(zsd_pset_t *pset, zsd_zone_t *zone)
{
        zsd_pset_usage_t *usage;

        for (usage = list_head(&pset->zsp_usage_list); usage != NULL;
            usage = list_next(&pset->zsp_usage_list, usage))
                if (usage->zsu_zone == zone)
                        return (usage);

        return (NULL);
}

static zsd_pset_usage_t *
zsd_allocate_pset_usage(zsd_ctl_t *ctl, zsd_pset_t *pset, zsd_zone_t *zone)
{
        zsd_pset_usage_t *usage;

        if ((usage = (zsd_pset_usage_t *)calloc(1, sizeof (zsd_pset_usage_t)))
            == NULL)
                return (NULL);

        list_link_init(&usage->zsu_next);
        usage->zsu_zone = zone;
        usage->zsu_zoneid = zone->zsz_id;
        usage->zsu_pset = pset;
        usage->zsu_found = B_FALSE;
        usage->zsu_active = B_FALSE;
        usage->zsu_new = B_FALSE;
        /*
         * Allocate as not deleted.  If a process is found in a pset for
         * a zone, the usage will not be deleted until at least the next
         * interval.
         */
        usage->zsu_start = g_now;
        usage->zsu_hrstart = g_hrnow;
        usage->zsu_deleted = B_FALSE;
        usage->zsu_empty = B_TRUE;
        usage->zsu_scheds = 0;
        usage->zsu_cpu_shares = ZS_LIMIT_NONE;

        ctl->zsctl_npset_usages++;
        pset->zsp_nusage++;

        return (usage);
}

static zsd_pset_usage_t *
zsd_lookup_insert_usage(zsd_ctl_t *ctl, zsd_pset_t *pset, zsd_zone_t *zone)
{
        zsd_pset_usage_t *usage, *tmp;

        if ((usage = zsd_lookup_usage(pset, zone))
            != NULL)
                return (usage);

        if ((usage = zsd_allocate_pset_usage(ctl, pset, zone)) == NULL)
                return (NULL);

        tmp = list_head(&pset->zsp_usage_list);
        while (tmp != NULL && strcmp(zone->zsz_name, tmp->zsu_zone->zsz_name)
            > 0)
                tmp = list_next(&pset->zsp_usage_list, tmp);

        list_insert_before(&pset->zsp_usage_list, tmp, usage);
        return (usage);
}

static void
zsd_refresh_system(zsd_ctl_t *ctl)
{
        zsd_system_t *system = ctl->zsctl_system;

        /* Re-count these values each interval */
        system->zss_processes = 0;
        system->zss_lwps = 0;
        system->zss_shm = 0;
        system->zss_shmids = 0;
        system->zss_semids = 0;
        system->zss_msgids = 0;
        system->zss_lofi = 0;
}


/* Reads each cpu's kstats, and adds the usage to the cpu's pset */
static void
zsd_update_cpu_stats(zsd_ctl_t *ctl, zsd_cpu_t *cpu)
{
        zsd_system_t *sys;
        processorid_t cpuid;
        zsd_pset_t *pset_prev;
        zsd_pset_t *pset;
        kstat_t *kstat;
        kstat_named_t *knp;
        kid_t kid;
        uint64_t idle, intr, kern, user;

        sys = ctl->zsctl_system;
        pset = cpu->zsc_pset;
        knp = NULL;
        kid = -1;
        cpuid = cpu->zsc_id;

        /* Get the cpu time totals for this cpu */
        kstat = kstat_lookup(ctl->zsctl_kstat_ctl, "cpu", cpuid, "sys");
        if (kstat == NULL)
                return;

        kid = kstat_read(ctl->zsctl_kstat_ctl, kstat, NULL);
        if (kid == -1)
                return;

        knp = kstat_data_lookup(kstat, "cpu_nsec_idle");
        if (knp == NULL || knp->data_type != KSTAT_DATA_UINT64)
                return;

        idle = knp->value.ui64;

        knp = kstat_data_lookup(kstat, "cpu_nsec_kernel");
        if (knp == NULL || knp->data_type != KSTAT_DATA_UINT64)
                return;

        kern = knp->value.ui64;

        knp = kstat_data_lookup(kstat, "cpu_nsec_user");
        if (knp == NULL || knp->data_type != KSTAT_DATA_UINT64)
                return;

        user = knp->value.ui64;

        /*
         * Tracking intr time per cpu just exists for future enhancements.
         * The value is presently always zero.
         */
        intr = 0;
        cpu->zsc_nsec_idle = idle;
        cpu->zsc_nsec_intr = intr;
        cpu->zsc_nsec_kern = kern;
        cpu->zsc_nsec_user = user;

        if (cpu->zsc_onlined == B_TRUE) {
                /*
                 * cpu is newly online.  There is no reference value,
                 * so just record its current stats for comparison
                 * on next stat read.
                 */
                cpu->zsc_nsec_idle_prev = cpu->zsc_nsec_idle;
                cpu->zsc_nsec_intr_prev = cpu->zsc_nsec_intr;
                cpu->zsc_nsec_kern_prev = cpu->zsc_nsec_kern;
                cpu->zsc_nsec_user_prev = cpu->zsc_nsec_user;
                return;
        }

        /*
         * Calculate relative time since previous refresh.
         * Paranoia.  Don't let time  go backwards.
         */
        idle = intr = kern = user = 0;
        if (cpu->zsc_nsec_idle > cpu->zsc_nsec_idle_prev)
                idle = cpu->zsc_nsec_idle - cpu->zsc_nsec_idle_prev;

        if (cpu->zsc_nsec_intr > cpu->zsc_nsec_intr_prev)
                intr = cpu->zsc_nsec_intr - cpu->zsc_nsec_intr_prev;

        if (cpu->zsc_nsec_kern > cpu->zsc_nsec_kern_prev)
                kern = cpu->zsc_nsec_kern - cpu->zsc_nsec_kern_prev;

        if (cpu->zsc_nsec_user > cpu->zsc_nsec_user_prev)
                user = cpu->zsc_nsec_user - cpu->zsc_nsec_user_prev;

        /* Update totals for cpu usage */
        TIMESTRUC_ADD_NANOSEC(cpu->zsc_idle, idle);
        TIMESTRUC_ADD_NANOSEC(cpu->zsc_intr, intr);
        TIMESTRUC_ADD_NANOSEC(cpu->zsc_kern, kern);
        TIMESTRUC_ADD_NANOSEC(cpu->zsc_user, user);

        /*
         * Add cpu's stats to its pset if it is known to be in
         * the pset since previous read.
         */
        if (cpu->zsc_psetid == cpu->zsc_psetid_prev ||
            cpu->zsc_psetid_prev == ZS_PSET_ERROR ||
            (pset_prev = zsd_lookup_pset_byid(ctl,
            cpu->zsc_psetid_prev)) == NULL) {
                TIMESTRUC_ADD_NANOSEC(pset->zsp_idle, idle);
                TIMESTRUC_ADD_NANOSEC(pset->zsp_intr, intr);
                TIMESTRUC_ADD_NANOSEC(pset->zsp_kern, kern);
                TIMESTRUC_ADD_NANOSEC(pset->zsp_user, user);
        } else {
                /*
                 * Last pset was different than current pset.
                 * Best guess is to split usage between the two.
                 */
                TIMESTRUC_ADD_NANOSEC(pset_prev->zsp_idle, idle / 2);
                TIMESTRUC_ADD_NANOSEC(pset_prev->zsp_intr, intr / 2);
                TIMESTRUC_ADD_NANOSEC(pset_prev->zsp_kern, kern / 2);
                TIMESTRUC_ADD_NANOSEC(pset_prev->zsp_user, user / 2);

                TIMESTRUC_ADD_NANOSEC(pset->zsp_idle,
                    (idle / 2) + (idle % 2));
                TIMESTRUC_ADD_NANOSEC(pset->zsp_intr,
                    (intr / 2) + (intr % 2));
                TIMESTRUC_ADD_NANOSEC(pset->zsp_kern,
                    (kern / 2) + (kern % 2));
                TIMESTRUC_ADD_NANOSEC(pset->zsp_user,
                    (user / 2) + (user % 2));
        }
        TIMESTRUC_ADD_NANOSEC(sys->zss_idle, idle);
        TIMESTRUC_ADD_NANOSEC(sys->zss_intr, intr);
        TIMESTRUC_ADD_NANOSEC(sys->zss_kern, kern);
        TIMESTRUC_ADD_NANOSEC(sys->zss_user, user);
}

/* Determine the details of a processor set by pset_id */
static int
zsd_get_pool_pset(zsd_ctl_t *ctl, psetid_t psetid, char *psetname,
    size_t namelen, uint_t *cputype, uint64_t *online, uint64_t *size,
    uint64_t *min, uint64_t *max, int64_t *importance)
{
        uint_t old, num;

        pool_conf_t *conf = ctl->zsctl_pool_conf;
        pool_value_t **vals = ctl->zsctl_pool_vals;
        pool_resource_t **res_list = NULL;
        pool_resource_t *pset;
        pool_component_t **cpus = NULL;
        processorid_t *cache;
        const char *string;
        uint64_t uint64;
        int64_t int64;
        int i, ret, type;

        if (ctl->zsctl_pool_status == POOL_DISABLED) {

                /*
                 * Inspect legacy psets
                 */
                for (;;) {
                        old = num = ctl->zsctl_cpu_ncache;
                        ret = pset_info(psetid, &type, &num,
                            ctl->zsctl_cpu_cache);
                        if (ret < 0) {
                                /* pset is gone.  Tell caller to retry */
                                errno = EINTR;
                                return (-1);
                        }
                        if (num <= old) {
                        /* Success */
                                break;
                        }
                        if ((cache = (processorid_t *)realloc(
                            ctl->zsctl_cpu_cache, num *
                            sizeof (processorid_t))) != NULL) {
                                ctl->zsctl_cpu_ncache = num;
                                ctl->zsctl_cpu_cache = cache;
                        } else {
                                /*
                                 * Could not allocate to get new cpu list.
                                 */
                                zsd_warn(gettext(
                                    "Could not allocate for cpu list"));
                                errno = ENOMEM;
                                return (-1);
                        }
                }
                /*
                 * Old school pset.  Just make min and max equal
                 * to its size
                 */
                if (psetid == ZS_PSET_DEFAULT) {
                        *cputype = ZS_CPUTYPE_DEFAULT_PSET;
                        (void) strlcpy(psetname, "pset_default", namelen);
                } else {
                        *cputype = ZS_CPUTYPE_PSRSET_PSET;
                        (void) snprintf(psetname, namelen,
                            "SUNWlegacy_pset_%d", psetid);
                }

                /*
                 * Just treat legacy pset as a simple pool pset
                 */
                *online = num;
                *size = num;
                *min = num;
                *max = num;
                *importance = 1;

                return (0);
        }

        /* Look up the pool pset using the pset id */
        res_list = NULL;
        pool_value_set_int64(vals[1], psetid);
        if (pool_value_set_name(vals[1], "pset.sys_id")
            != PO_SUCCESS)
                goto err;

        if (pool_value_set_name(vals[0], "type") != PO_SUCCESS)
                goto err;
        if (pool_value_set_string(vals[0], "pset") != PO_SUCCESS)
                goto err;
        if ((res_list = pool_query_resources(conf, &num, vals)) == NULL)
                goto err;
        if (num != 1)
                goto err;
        pset = res_list[0];
        free(res_list);
        res_list = NULL;
        if (pool_get_property(conf, pool_resource_to_elem(conf, pset),
            "pset.name", vals[0]) != POC_STRING ||
            pool_value_get_string(vals[0], &string) != PO_SUCCESS)
                goto err;

        (void) strlcpy(psetname, string, namelen);
        if (strncmp(psetname, "SUNWtmp", strlen("SUNWtmp")) == 0)
                *cputype = ZS_CPUTYPE_DEDICATED;
        else if (psetid == ZS_PSET_DEFAULT)
                *cputype = ZS_CPUTYPE_DEFAULT_PSET;
        else
                *cputype = ZS_CPUTYPE_POOL_PSET;

        /* Get size, min, max, and importance */
        if (pool_get_property(conf, pool_resource_to_elem(conf,
            pset), "pset.size", vals[0]) == POC_UINT &&
            pool_value_get_uint64(vals[0], &uint64) == PO_SUCCESS)
                *size = uint64;
        else
                *size = 0;

                /* Get size, min, max, and importance */
        if (pool_get_property(conf, pool_resource_to_elem(conf,
            pset), "pset.min", vals[0]) == POC_UINT &&
            pool_value_get_uint64(vals[0], &uint64) == PO_SUCCESS)
                *min = uint64;
        else
                *min = 0;
        if (*min >= ZSD_PSET_UNLIMITED)
                *min = ZS_LIMIT_NONE;

        if (pool_get_property(conf, pool_resource_to_elem(conf,
            pset), "pset.max", vals[0]) == POC_UINT &&
            pool_value_get_uint64(vals[0], &uint64) == PO_SUCCESS)
                *max = uint64;
        else
                *max = ZS_LIMIT_NONE;

        if (*max >= ZSD_PSET_UNLIMITED)
                *max = ZS_LIMIT_NONE;

        if (pool_get_property(conf, pool_resource_to_elem(conf,
            pset), "pset.importance", vals[0]) == POC_INT &&
            pool_value_get_int64(vals[0], &int64) == PO_SUCCESS)
                *importance = int64;
        else
                *importance = (uint64_t)1;

        *online = 0;
        if (*size == 0)
                return (0);

        /* get cpus */
        cpus = pool_query_resource_components(conf, pset, &num, NULL);
        if (cpus == NULL)
                goto err;

        /* Make sure there is space for cpu id list */
        if (num > ctl->zsctl_cpu_ncache) {
                if ((cache = (processorid_t *)realloc(
                    ctl->zsctl_cpu_cache, num *
                    sizeof (processorid_t))) != NULL) {
                        ctl->zsctl_cpu_ncache = num;
                        ctl->zsctl_cpu_cache = cache;
                } else {
                        /*
                         * Could not allocate to get new cpu list.
                         */
                        zsd_warn(gettext(
                            "Could not allocate for cpu list"));
                        goto err;
                }
        }

        /* count the online cpus */
        for (i = 0; i < num; i++) {
                if (pool_get_property(conf, pool_component_to_elem(
                    conf, cpus[i]), "cpu.status", vals[0]) != POC_STRING ||
                    pool_value_get_string(vals[0], &string) != PO_SUCCESS)
                        goto err;

                if (strcmp(string, "on-line") != 0 &&
                    strcmp(string, "no-intr") != 0)
                        continue;

                if (pool_get_property(conf, pool_component_to_elem(
                    conf, cpus[i]), "cpu.sys_id", vals[0]) != POC_INT ||
                    pool_value_get_int64(vals[0], &int64) != PO_SUCCESS)
                        goto err;

                (*online)++;
                ctl->zsctl_cpu_cache[i] = (psetid_t)int64;
        }
        free(cpus);
        return (0);
err:
        free(res_list);
        free(cpus);

        /*
         * The pools operations should succeed since the conf is a consistent
         * snapshot.  Tell caller there is no need to retry.
         */
        errno = EINVAL;
        return (-1);
}

/*
 * Update the current list of processor sets.
 * This also updates the list of online cpus, and each cpu's pset membership.
 */
static void
zsd_refresh_psets(zsd_ctl_t *ctl)
{
        int i, j, ret, state;
        uint_t old, num;
        uint_t cputype;
        int64_t sys_id, importance;
        uint64_t online, size, min, max;
        zsd_system_t *system;
        zsd_pset_t *pset;
        zsd_cpu_t *cpu;
        psetid_t *cache;
        char psetname[ZS_PSETNAME_MAX];
        processorid_t cpuid;
        pool_value_t *pv_save = NULL;
        pool_resource_t **res_list = NULL;
        pool_resource_t *res;
        pool_value_t **vals;
        pool_conf_t *conf;
        boolean_t roll_cpus = B_TRUE;

        /* Zero cpu counters to recount them */
        system = ctl->zsctl_system;
        system->zss_ncpus = 0;
        system->zss_ncpus_online = 0;
retry:
        ret = pool_get_status(&state);
        if (ret == 0 && state == POOL_ENABLED) {

                conf = ctl->zsctl_pool_conf;
                vals = ctl->zsctl_pool_vals;
                pv_save = vals[1];
                vals[1] = NULL;

                if (ctl->zsctl_pool_status == POOL_DISABLED) {
                        if (pool_conf_open(ctl->zsctl_pool_conf,
                            pool_dynamic_location(), PO_RDONLY) == 0) {
                                ctl->zsctl_pool_status = POOL_ENABLED;
                                ctl->zsctl_pool_changed = POU_PSET;
                        }
                } else {
                        ctl->zsctl_pool_changed = 0;
                        ret = pool_conf_update(ctl->zsctl_pool_conf,
                            &(ctl->zsctl_pool_changed));
                        if (ret < 0) {
                                /* Pools must have become disabled */
                                (void) pool_conf_close(ctl->zsctl_pool_conf);
                                ctl->zsctl_pool_status = POOL_DISABLED;
                                if (pool_error() == POE_SYSTEM && errno ==
                                    ENOTACTIVE)
                                        goto retry;

                                zsd_warn(gettext(
                                    "Unable to update pool configuration"));
                                /* Not able to get pool info.  Don't update. */
                                goto err;
                        }
                }
                /* Get the list of psets using libpool */
                if (pool_value_set_name(vals[0], "type") != PO_SUCCESS)
                        goto err;

                if (pool_value_set_string(vals[0], "pset") != PO_SUCCESS)
                        goto err;
                if ((res_list = pool_query_resources(conf, &num, vals))
                    == NULL)
                        goto err;

                if (num > ctl->zsctl_pset_ncache)  {
                        if ((cache = (psetid_t *)realloc(ctl->zsctl_pset_cache,
                            (num) * sizeof (psetid_t))) == NULL) {
                                goto err;
                        }
                        ctl->zsctl_pset_ncache = num;
                        ctl->zsctl_pset_cache = cache;
                }
                /* Save the pset id of each pset */
                for (i = 0; i < num; i++) {
                        res = res_list[i];
                        if (pool_get_property(conf, pool_resource_to_elem(conf,
                            res), "pset.sys_id", vals[0]) != POC_INT ||
                            pool_value_get_int64(vals[0], &sys_id)
                            != PO_SUCCESS)
                                goto err;
                        ctl->zsctl_pset_cache[i] = (int)sys_id;
                }
                vals[1] = pv_save;
                pv_save = NULL;
        } else {
                if (ctl->zsctl_pool_status == POOL_ENABLED) {
                        (void) pool_conf_close(ctl->zsctl_pool_conf);
                        ctl->zsctl_pool_status = POOL_DISABLED;
                }
                /* Get the pset list using legacy psets */
                for (;;) {
                        old = num = ctl->zsctl_pset_ncache;
                        (void) pset_list(ctl->zsctl_pset_cache, &num);
                        if ((num + 1) <= old) {
                                break;
                        }
                        if ((cache = (psetid_t *)realloc(ctl->zsctl_pset_cache,
                            (num + 1) * sizeof (psetid_t))) != NULL) {
                                ctl->zsctl_pset_ncache = num + 1;
                                ctl->zsctl_pset_cache = cache;
                        } else {
                                /*
                                 * Could not allocate to get new pset list.
                                 * Give up
                                 */
                                return;
                        }
                }
                /* Add the default pset to list */
                ctl->zsctl_pset_cache[num] = ctl->zsctl_pset_cache[0];
                ctl->zsctl_pset_cache[0] = ZS_PSET_DEFAULT;
                num++;
        }
psets_changed:
        zsd_mark_cpus_start(ctl, roll_cpus);
        zsd_mark_psets_start(ctl);
        roll_cpus = B_FALSE;

        /* Refresh cpu membership of all psets */
        for (i = 0; i < num; i++) {

                /* Get pool pset information */
                sys_id = ctl->zsctl_pset_cache[i];
                if (zsd_get_pool_pset(ctl, sys_id, psetname, sizeof (psetname),
                    &cputype, &online, &size, &min, &max, &importance)
                    != 0) {
                        if (errno == EINTR)
                                goto psets_changed;
                        zsd_warn(gettext("Failed to get info for pset %d"),
                            sys_id);
                        continue;
                }

                system->zss_ncpus += size;
                system->zss_ncpus_online += online;

                pset = zsd_lookup_insert_pset(ctl, psetname,
                    ctl->zsctl_pset_cache[i]);

                /* update pset info */
                zsd_mark_pset_found(pset, cputype, online, size, min,
                    max, importance);

                /* update each cpu in pset */
                for (j = 0; j < pset->zsp_online; j++) {
                        cpuid = ctl->zsctl_cpu_cache[j];
                        cpu = zsd_lookup_insert_cpu(ctl, cpuid);
                        zsd_mark_cpu_found(cpu, pset, sys_id);
                }
        }
err:
        free(res_list);
        if (pv_save != NULL)
                vals[1] = pv_save;
}



/*
 * Fetch the current pool and pset name for the given zone.
 */
static void
zsd_get_zone_pool_pset(zsd_ctl_t *ctl, zsd_zone_t *zone,
    char *pool, int poollen, char *pset, int psetlen, uint_t *cputype)
{
        poolid_t poolid;
        pool_t **pools = NULL;
        pool_resource_t **res_list = NULL;
        char poolname[ZS_POOLNAME_MAX];
        char psetname[ZS_PSETNAME_MAX];
        pool_conf_t *conf = ctl->zsctl_pool_conf;
        pool_value_t *pv_save = NULL;
        pool_value_t **vals = ctl->zsctl_pool_vals;
        const char *string;
        int ret;
        int64_t int64;
        uint_t num;

        ret = zone_getattr(zone->zsz_id, ZONE_ATTR_POOLID,
            &poolid, sizeof (poolid));
        if (ret < 0)
                goto lookup_done;

        pv_save = vals[1];
        vals[1] = NULL;
        pools = NULL;
        res_list = NULL;

        /* Default values if lookup fails */
        (void) strlcpy(poolname, "pool_default", sizeof (poolname));
        (void) strlcpy(psetname, "pset_default", sizeof (poolname));
        *cputype = ZS_CPUTYPE_DEFAULT_PSET;

        /* no dedicated cpu if pools are disabled */
        if (ctl->zsctl_pool_status == POOL_DISABLED)
                goto lookup_done;

        /* Get the pool name using the id */
        pool_value_set_int64(vals[0], poolid);
        if (pool_value_set_name(vals[0], "pool.sys_id") != PO_SUCCESS)
                goto lookup_done;

        if ((pools = pool_query_pools(conf, &num, vals)) == NULL)
                goto lookup_done;

        if (num != 1)
                goto lookup_done;

        if (pool_get_property(conf, pool_to_elem(conf, pools[0]),
            "pool.name", vals[0]) != POC_STRING ||
            pool_value_get_string(vals[0], &string) != PO_SUCCESS)
                goto lookup_done;
        (void) strlcpy(poolname, (char *)string, sizeof (poolname));

        /* Get the name of the pset for the pool */
        if (pool_value_set_name(vals[0], "type") != PO_SUCCESS)
                goto lookup_done;

        if (pool_value_set_string(vals[0], "pset") != PO_SUCCESS)
                goto lookup_done;

        if ((res_list = pool_query_pool_resources(conf, pools[0], &num, vals))
            == NULL)
                goto lookup_done;

        if (num != 1)
                goto lookup_done;

        if (pool_get_property(conf, pool_resource_to_elem(conf,
            res_list[0]), "pset.sys_id", vals[0]) != POC_INT ||
            pool_value_get_int64(vals[0], &int64) != PO_SUCCESS)
                goto lookup_done;

        if (int64 == ZS_PSET_DEFAULT)
                *cputype = ZS_CPUTYPE_DEFAULT_PSET;

        if (pool_get_property(conf, pool_resource_to_elem(conf,
            res_list[0]), "pset.name", vals[0]) != POC_STRING ||
            pool_value_get_string(vals[0], &string) != PO_SUCCESS)
                goto lookup_done;

        (void) strlcpy(psetname, (char *)string, sizeof (psetname));

        if (strncmp(psetname, "SUNWtmp_", strlen("SUNWtmp_")) == 0)
                *cputype = ZS_CPUTYPE_DEDICATED;
        if (strncmp(psetname, "SUNW_legacy_", strlen("SUNW_legacy_")) == 0)
                *cputype = ZS_CPUTYPE_PSRSET_PSET;
        else
                *cputype = ZS_CPUTYPE_POOL_PSET;

lookup_done:

        if (pv_save != NULL)
                vals[1] = pv_save;

        free(res_list);
        free(pools);

        (void) strlcpy(pool, poolname, poollen);
        (void) strlcpy(pset, psetname, psetlen);
}

/* Convert scheduler names to ZS_* scheduler flags */
static uint_t
zsd_schedname2int(char *clname, int pri)
{
        uint_t sched = 0;

        if (strcmp(clname, "TS") == 0) {
                sched = ZS_SCHED_TS;
        } else if (strcmp(clname, "IA") == 0) {
                sched = ZS_SCHED_IA;
        } else if (strcmp(clname, "FX") == 0) {
                if (pri > 59) {
                        sched = ZS_SCHED_FX_60;
                } else {
                        sched = ZS_SCHED_FX;
                }
        } else if (strcmp(clname, "RT") == 0) {
                sched = ZS_SCHED_RT;

        } else if (strcmp(clname, "FSS") == 0) {
                sched = ZS_SCHED_FSS;
        }
        return (sched);
}

static uint64_t
zsd_get_zone_rctl_limit(char *name)
{
        rctlblk_t *rblk;

        rblk = (rctlblk_t *)alloca(rctlblk_size());
        if (getrctl(name, NULL, rblk, RCTL_FIRST)
            != 0) {
                return (ZS_LIMIT_NONE);
        }
        return (rctlblk_get_value(rblk));
}

static uint64_t
zsd_get_zone_rctl_usage(char *name)
{
        rctlblk_t *rblk;

        rblk = (rctlblk_t *)alloca(rctlblk_size());
        if (getrctl(name, NULL, rblk, RCTL_USAGE)
            != 0) {
                return (0);
        }
        return (rctlblk_get_value(rblk));
}

#define ZSD_NUM_RCTL_VALS 19

/*
 * Fetch the limit information for a zone.  This uses zone_enter() as the
 * getrctl(2) system call only returns rctl information for the zone of
 * the caller.
 */
static int
zsd_get_zone_caps(zsd_ctl_t *ctl, zsd_zone_t *zone, uint64_t *cpu_shares,
    uint64_t *cpu_cap, uint64_t *ram_cap, uint64_t *locked_cap,
    uint64_t *vm_cap, uint64_t *processes_cap, uint64_t *processes,
    uint64_t *lwps_cap, uint64_t *lwps, uint64_t *shm_cap, uint64_t *shm,
    uint64_t *shmids_cap, uint64_t *shmids, uint64_t *semids_cap,
    uint64_t *semids, uint64_t *msgids_cap, uint64_t *msgids,
    uint64_t *lofi_cap, uint64_t *lofi, uint_t *sched)
{
        int p[2], pid, tmpl_fd, ret;
        ctid_t ct;
        char class[PC_CLNMSZ];
        uint64_t vals[ZSD_NUM_RCTL_VALS];
        zsd_system_t *sys = ctl->zsctl_system;
        int i = 0;
        int res = 0;

        /* Treat all caps as no cap on error */
        *cpu_shares = ZS_LIMIT_NONE;
        *cpu_cap = ZS_LIMIT_NONE;
        *ram_cap = ZS_LIMIT_NONE;
        *locked_cap = ZS_LIMIT_NONE;
        *vm_cap = ZS_LIMIT_NONE;

        *processes_cap = ZS_LIMIT_NONE;
        *lwps_cap = ZS_LIMIT_NONE;
        *shm_cap = ZS_LIMIT_NONE;
        *shmids_cap = ZS_LIMIT_NONE;
        *semids_cap = ZS_LIMIT_NONE;
        *msgids_cap = ZS_LIMIT_NONE;
        *lofi_cap = ZS_LIMIT_NONE;

        *processes = 0;
        *lwps = 0;
        *shm = 0;
        *shmids = 0;
        *semids = 0;
        *msgids = 0;
        *lofi = 0;

        /* Get the ram cap first since it is a zone attr */
        ret = zone_getattr(zone->zsz_id, ZONE_ATTR_PHYS_MCAP,
            ram_cap, sizeof (*ram_cap));
        if (ret < 0 || *ram_cap == 0)
                *ram_cap = ZS_LIMIT_NONE;

        /* Get the zone's default scheduling class */
        ret = zone_getattr(zone->zsz_id, ZONE_ATTR_SCHED_CLASS,
            class, sizeof (class));
        if (ret < 0)
                return (-1);

        *sched = zsd_schedname2int(class, 0);

        /* rctl caps must be fetched from within the zone */
        if (pipe(p) != 0)
                return (-1);

        if ((tmpl_fd = init_template()) == -1) {
                (void) close(p[0]);
                (void) close(p[1]);
                return (-1);
        }
        pid = forkx(0);
        if (pid < 0) {
                (void) ct_tmpl_clear(tmpl_fd);
                (void) close(p[0]);
                (void) close(p[1]);
                return (-1);
        }
        if (pid == 0) {

                (void) ct_tmpl_clear(tmpl_fd);
                (void) close(tmpl_fd);
                (void) close(p[0]);
                if (zone->zsz_id != getzoneid()) {
                        if (zone_enter(zone->zsz_id) < 0) {
                                (void) close(p[1]);
                                _exit(0);
                        }
                }

                /* Get caps for zone, and write them to zonestatd parent. */
                vals[i++] = zsd_get_zone_rctl_limit("zone.cpu-shares");
                vals[i++] = zsd_get_zone_rctl_limit("zone.cpu-cap");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-locked-memory");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-swap");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-processes");
                vals[i++] = zsd_get_zone_rctl_usage("zone.max-processes");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-lwps");
                vals[i++] = zsd_get_zone_rctl_usage("zone.max-lwps");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-shm-memory");
                vals[i++] = zsd_get_zone_rctl_usage("zone.max-shm-memory");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-shm-ids");
                vals[i++] = zsd_get_zone_rctl_usage("zone.max-shm-ids");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-sem-ids");
                vals[i++] = zsd_get_zone_rctl_usage("zone.max-sem-ids");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-msg-ids");
                vals[i++] = zsd_get_zone_rctl_usage("zone.max-msg-ids");
                vals[i++] = zsd_get_zone_rctl_limit("zone.max-lofi");
                vals[i++] = zsd_get_zone_rctl_usage("zone.max-lofi");

                if (write(p[1], vals, ZSD_NUM_RCTL_VALS * sizeof (uint64_t)) !=
                    ZSD_NUM_RCTL_VALS * sizeof (uint64_t)) {
                        (void) close(p[1]);
                        _exit(1);
                }

                (void) close(p[1]);
                _exit(0);
        }
        if (contract_latest(&ct) == -1)
                ct = -1;

        (void) ct_tmpl_clear(tmpl_fd);
        (void) close(tmpl_fd);
        (void) close(p[1]);
        while (waitpid(pid, NULL, 0) != pid)
                ;

        /* Read cap from child in zone */
        if (read(p[0], vals, ZSD_NUM_RCTL_VALS * sizeof (uint64_t)) !=
            ZSD_NUM_RCTL_VALS * sizeof (uint64_t)) {
                res = -1;
                goto cleanup;
        }
        i = 0;
        *cpu_shares = vals[i++];
        *cpu_cap = vals[i++];
        *locked_cap = vals[i++];
        *vm_cap = vals[i++];
        *processes_cap = vals[i++];
        *processes = vals[i++];
        *lwps_cap = vals[i++];
        *lwps = vals[i++];
        *shm_cap = vals[i++];
        *shm = vals[i++];
        *shmids_cap = vals[i++];
        *shmids = vals[i++];
        *semids_cap = vals[i++];
        *semids = vals[i++];
        *msgids_cap = vals[i++];
        *msgids = vals[i++];
        *lofi_cap = vals[i++];
        *lofi = vals[i++];

        /* Interpret maximum values as no cap */
        if (*cpu_cap == UINT32_MAX || *cpu_cap == 0)
                *cpu_cap = ZS_LIMIT_NONE;
        if (*processes_cap == sys->zss_processes_max)
                *processes_cap = ZS_LIMIT_NONE;
        if (*lwps_cap == sys->zss_lwps_max)
                *lwps_cap = ZS_LIMIT_NONE;
        if (*shm_cap == sys->zss_shm_max)
                *shm_cap = ZS_LIMIT_NONE;
        if (*shmids_cap == sys->zss_shmids_max)
                *shmids_cap = ZS_LIMIT_NONE;
        if (*semids_cap == sys->zss_semids_max)
                *semids_cap = ZS_LIMIT_NONE;
        if (*msgids_cap == sys->zss_msgids_max)
                *msgids_cap = ZS_LIMIT_NONE;
        if (*lofi_cap == sys->zss_lofi_max)
                *lofi_cap = ZS_LIMIT_NONE;


cleanup:
        (void) close(p[0]);
        (void) ct_tmpl_clear(tmpl_fd);
        (void) close(tmpl_fd);
        (void) contract_abandon_id(ct);

        return (res);
}

/* Update the current list of running zones */
static void
zsd_refresh_zones(zsd_ctl_t *ctl)
{
        zsd_zone_t *zone;
        uint_t old, num;
        ushort_t flags;
        int i, ret;
        zoneid_t *cache;
        uint64_t cpu_shares;
        uint64_t cpu_cap;
        uint64_t ram_cap;
        uint64_t locked_cap;
        uint64_t vm_cap;
        uint64_t processes_cap;
        uint64_t processes;
        uint64_t lwps_cap;
        uint64_t lwps;
        uint64_t shm_cap;
        uint64_t shm;
        uint64_t shmids_cap;
        uint64_t shmids;
        uint64_t semids_cap;
        uint64_t semids;
        uint64_t msgids_cap;
        uint64_t msgids;
        uint64_t lofi_cap;
        uint64_t lofi;

        char zonename[ZS_ZONENAME_MAX];
        char poolname[ZS_POOLNAME_MAX];
        char psetname[ZS_PSETNAME_MAX];
        uint_t sched;
        uint_t cputype;
        uint_t iptype;

        /* Get the current list of running zones */
        for (;;) {
                old = num = ctl->zsctl_zone_ncache;
                (void) zone_list(ctl->zsctl_zone_cache, &num);
                if (num <= old)
                        break;
                if ((cache = (zoneid_t *)realloc(ctl->zsctl_zone_cache,
                    (num) * sizeof (zoneid_t))) != NULL) {
                        ctl->zsctl_zone_ncache = num;
                        ctl->zsctl_zone_cache = cache;
                } else {
                        /* Could not allocate to get new zone list.  Give up */
                        return;
                }
        }

        zsd_mark_zones_start(ctl);

        for (i = 0; i < num; i++) {

                ret = getzonenamebyid(ctl->zsctl_zone_cache[i],
                    zonename, sizeof (zonename));
                if (ret < 0)
                        continue;

                zone = zsd_lookup_insert_zone(ctl, zonename,
                    ctl->zsctl_zone_cache[i]);

                ret = zone_getattr(ctl->zsctl_zone_cache[i], ZONE_ATTR_FLAGS,
                    &flags, sizeof (flags));
                if (ret < 0)
                        continue;

                if (flags & ZF_NET_EXCL)
                        iptype = ZS_IPTYPE_EXCLUSIVE;
                else
                        iptype = ZS_IPTYPE_SHARED;

                zsd_get_zone_pool_pset(ctl, zone, poolname, sizeof (poolname),
                    psetname, sizeof (psetname), &cputype);

                if (zsd_get_zone_caps(ctl, zone, &cpu_shares, &cpu_cap,
                    &ram_cap, &locked_cap, &vm_cap, &processes_cap, &processes,
                    &lwps_cap, &lwps, &shm_cap, &shm, &shmids_cap, &shmids,
                    &semids_cap, &semids, &msgids_cap, &msgids, &lofi_cap,
                    &lofi, &sched) != 0)
                        continue;

                zsd_mark_zone_found(ctl, zone, cpu_shares, cpu_cap, ram_cap,
                    locked_cap, vm_cap, processes_cap, processes, lwps_cap,
                    lwps, shm_cap, shm, shmids_cap, shmids, semids_cap,
                    semids, msgids_cap, msgids, lofi_cap, lofi, poolname,
                    psetname, sched, cputype, iptype);
        }
}

/* Fetch the details of a process from its psinfo_t */
static void
zsd_get_proc_info(zsd_ctl_t *ctl, psinfo_t *psinfo, psetid_t *psetid,
    psetid_t *prev_psetid, zoneid_t *zoneid, zoneid_t *prev_zoneid,
    timestruc_t *delta, uint_t *sched)
{
        timestruc_t d;
        zsd_proc_t *proc;

        /* Get cached data for proc */
        proc = &(ctl->zsctl_proc_array[psinfo->pr_pid]);
        *psetid = psinfo->pr_lwp.pr_bindpset;

        if (proc->zspr_psetid == ZS_PSET_ERROR)
                *prev_psetid = *psetid;
        else
                *prev_psetid = proc->zspr_psetid;

        *zoneid = psinfo->pr_zoneid;
        if (proc->zspr_zoneid == -1)
                *prev_zoneid = *zoneid;
        else
                *prev_zoneid = proc->zspr_zoneid;

        TIMESTRUC_DELTA(d, psinfo->pr_time, proc->zspr_usage);
        *delta = d;

        *sched = zsd_schedname2int(psinfo->pr_lwp.pr_clname,
            psinfo->pr_lwp.pr_pri);

        /* Update cached data for proc */
        proc->zspr_psetid = psinfo->pr_lwp.pr_bindpset;
        proc->zspr_zoneid = psinfo->pr_zoneid;
        proc->zspr_sched = *sched;
        proc->zspr_usage.tv_sec = psinfo->pr_time.tv_sec;
        proc->zspr_usage.tv_nsec = psinfo->pr_time.tv_nsec;
        proc->zspr_ppid = psinfo->pr_ppid;
}

/*
 * Reset the known cpu usage of a process. This is done after a process
 * exits so that if the pid is recycled, data from its previous life is
 * not reused
 */
static void
zsd_flush_proc_info(zsd_proc_t *proc)
{
        proc->zspr_usage.tv_sec = 0;
        proc->zspr_usage.tv_nsec = 0;
}

/*
 * Open the current extended accounting file.  On initialization, open the
 * file as the current file to be used.  Otherwise, open the file as the
 * next file to use of the current file reaches EOF.
 */
static int
zsd_open_exacct(zsd_ctl_t *ctl, boolean_t init)
{
        int ret, oret, state, trys = 0, flags;
        int *fd, *open;
        ea_file_t *eaf;
        struct stat64 *stat;
        char path[MAXPATHLEN];

        /*
         * The accounting file is first opened at the tail.  Following
         * opens to new accounting files are opened at the head.
         */
        if (init == B_TRUE) {
                flags = EO_NO_VALID_HDR | EO_TAIL;
                fd = &ctl->zsctl_proc_fd;
                eaf = &ctl->zsctl_proc_eaf;
                stat = &ctl->zsctl_proc_stat;
                open = &ctl->zsctl_proc_open;
        } else {
                flags = EO_NO_VALID_HDR | EO_HEAD;
                fd = &ctl->zsctl_proc_fd_next;
                eaf = &ctl->zsctl_proc_eaf_next;
                stat = &ctl->zsctl_proc_stat_next;
                open = &ctl->zsctl_proc_open_next;
        }

        *fd = -1;
        *open = 0;
retry:
        /* open accounting files for cpu consumption */
        ret = acctctl(AC_STATE_GET | AC_PROC, &state, sizeof (state));
        if (ret != 0) {
                zsd_warn(gettext("Unable to get process accounting state"));
                goto err;
        }
        if (state != AC_ON) {
                if (trys > 0) {
                        zsd_warn(gettext(
                            "Unable to enable process accounting"));
                        goto err;
                }
                (void) zsd_enable_cpu_stats();
                trys++;
                goto retry;
        }

        ret = acctctl(AC_FILE_GET | AC_PROC, path, sizeof (path));
        if (ret != 0) {
                zsd_warn(gettext("Unable to get process accounting file"));
                goto err;
        }

        if ((*fd = open64(path, O_RDONLY, 0)) >= 0 &&
            (oret = ea_fdopen(eaf, *fd, NULL, flags, O_RDONLY)) == 0)
                ret = fstat64(*fd, stat);

        if (*fd < 0 || oret < 0 || ret < 0) {
                struct timespec ts;

                /*
                 * It is possible the accounting file is momentarily unavailable
                 * because it is being rolled.  Try for up to half a second.
                 *
                 * If failure to open accounting file persists, give up.
                 */
                if (oret == 0)
                        (void) ea_close(eaf);
                else if (*fd >= 0)
                        (void) close(*fd);
                if (trys > 500) {
                        zsd_warn(gettext(
                            "Unable to open process accounting file"));
                        goto err;
                }
                /* wait one millisecond */
                ts.tv_sec = 0;
                ts.tv_nsec = NANOSEC / 1000;
                (void) nanosleep(&ts, NULL);
                goto retry;
        }
        *open = 1;
        return (0);
err:
        if (*fd >= 0)
                (void) close(*fd);
        *open = 0;
        *fd = -1;
        return (-1);
}

/*
 * Walk /proc and charge each process to its zone and processor set.
 * Then read exacct data for exited processes, and charge them as well.
 */
static void
zsd_refresh_procs(zsd_ctl_t *ctl, boolean_t init)
{
        DIR *dir;
        struct dirent *dent, *dresult;
        psinfo_t psinfo;
        int fd, ret;
        zsd_proc_t *proc, *pproc, *tmp, *next;
        list_t pplist, plist;
        zsd_zone_t *zone, *prev_zone;
        zsd_pset_t *pset, *prev_pset;
        psetid_t psetid, prev_psetid;
        zoneid_t zoneid, prev_zoneid;
        zsd_pset_usage_t *usage, *prev_usage;
        char path[MAXPATHLEN];

        ea_object_t object;
        ea_object_t pobject;
        boolean_t hrtime_expired = B_FALSE;
        struct timeval interval_end;

        timestruc_t delta, d1, d2;
        uint_t sched = 0;

        /*
         * Get the current accounting file.  The current accounting file
         * may be different than the file in use, as the accounting file
         * may have been rolled, or manually changed by an admin.
         */
        ret = zsd_open_exacct(ctl, init);
        if (ret != 0) {
                zsd_warn(gettext("Unable to track process accounting"));
                return;
        }

        /*
         * Mark the current time as the interval end time.  Don't track
         * processes that exit after this time.
         */
        (void) gettimeofday(&interval_end, NULL);

        dir = opendir("/proc");
        if (dir == NULL) {
                zsd_warn(gettext("Unable to open /proc"));
                return;
        }

        dent = ctl->zsctl_procfs_dent;

        (void) memset(dent, 0, ctl->zsctl_procfs_dent_size);

        /* Walk all processes and compute each zone's usage on each pset. */
        while (readdir_r(dir, dent, &dresult) == 0) {

                if (strcmp(dent->d_name, ".") == 0 ||
                    strcmp(dent->d_name, "..") == 0)
                        continue;

                (void) snprintf(path, sizeof (path), "/proc/%s/psinfo",
                    dent->d_name);

                fd = open(path, O_RDONLY);
                if (fd < 0)
                        continue;

                if (read(fd, &psinfo, sizeof (psinfo)) != sizeof (psinfo)) {
                        (void) close(fd);
                        continue;
                }
                (void) close(fd);

                zsd_get_proc_info(ctl, &psinfo, &psetid, &prev_psetid,
                    &zoneid, &prev_zoneid, &delta, &sched);

                d1.tv_sec = delta.tv_sec / 2;
                d1.tv_nsec = delta.tv_nsec / 2;
                d2.tv_sec = (delta.tv_sec / 2) + (delta.tv_sec % 2);
                d2.tv_nsec = (delta.tv_nsec / 2) + (delta.tv_nsec % 2);

                /* Get the zone and pset this process is running in */
                zone = zsd_lookup_zone_byid(ctl, zoneid);
                if (zone == NULL)
                        continue;
                pset = zsd_lookup_pset_byid(ctl, psetid);
                if (pset == NULL)
                        continue;
                usage = zsd_lookup_insert_usage(ctl, pset, zone);
                if (usage == NULL)
                        continue;

                /*
                 * Get the usage of the previous zone and pset if they were
                 * different.
                 */
                if (zoneid != prev_zoneid)
                        prev_zone = zsd_lookup_zone_byid(ctl, prev_zoneid);
                else
                        prev_zone = NULL;

                if (psetid != prev_psetid)
                        prev_pset = zsd_lookup_pset_byid(ctl, prev_psetid);
                else
                        prev_pset = NULL;

                prev_usage = NULL;
                if (prev_zone != NULL || prev_pset != NULL) {
                        if (prev_zone == NULL)
                                prev_zone = zone;
                        if (prev_pset == NULL)
                                prev_pset = pset;

                        prev_usage = zsd_lookup_insert_usage(ctl, prev_pset,
                            prev_zone);
                }

                /* Update the usage with the processes info */
                if (prev_usage == NULL) {
                        zsd_mark_pset_usage_found(usage, sched);
                } else {
                        zsd_mark_pset_usage_found(usage, sched);
                        zsd_mark_pset_usage_found(prev_usage, sched);
                }

                /*
                 * First time around is just to get a starting point.  All
                 * usages will be zero.
                 */
                if (init == B_TRUE)
                        continue;

                if (prev_usage == NULL) {
                        zsd_add_usage(ctl, usage, &delta);
                } else {
                        zsd_add_usage(ctl, usage, &d1);
                        zsd_add_usage(ctl, prev_usage, &d2);
                }
        }
        (void) closedir(dir);

        /*
         * No need to collect exited proc data on initialization.  Just
         * caching the usage of the known processes to get a zero starting
         * point.
         */
        if (init == B_TRUE)
                return;

        /*
         * Add accounting records to account for processes which have
         * exited.
         */
        list_create(&plist, sizeof (zsd_proc_t),
            offsetof(zsd_proc_t, zspr_next));
        list_create(&pplist, sizeof (zsd_proc_t),
            offsetof(zsd_proc_t, zspr_next));

        for (;;) {
                pid_t pid;
                pid_t ppid;
                timestruc_t user, sys, proc_usage;
                timestruc_t finish;
                int numfound = 0;

                bzero(&object, sizeof (object));
                proc = NULL;
                zone = NULL;
                pset = NULL;
                usage = NULL;
                ret = ea_get_object(&ctl->zsctl_proc_eaf, &object);
                if (ret == EO_ERROR) {
                        if (ea_error() == EXR_EOF) {

                                struct stat64 *stat;
                                struct stat64 *stat_next;

                                /*
                                 * See if the next accounting file is the
                                 * same as the current accounting file.
                                 */
                                stat = &(ctl->zsctl_proc_stat);
                                stat_next = &(ctl->zsctl_proc_stat_next);
                                if (stat->st_ino == stat_next->st_ino &&
                                    stat->st_dev == stat_next->st_dev) {
                                        /*
                                         * End of current accounting file is
                                         * reached, so finished.  Clear EOF
                                         * bit for next time around.
                                         */
                                        ea_clear(&ctl->zsctl_proc_eaf);
                                        break;
                                } else {
                                        /*
                                         * Accounting file has changed.  Move
                                         * to current accounting file.
                                         */
                                        (void) ea_close(&ctl->zsctl_proc_eaf);

                                        ctl->zsctl_proc_fd =
                                            ctl->zsctl_proc_fd_next;
                                        ctl->zsctl_proc_eaf =
                                            ctl->zsctl_proc_eaf_next;
                                        ctl->zsctl_proc_stat =
                                            ctl->zsctl_proc_stat_next;

                                        ctl->zsctl_proc_fd_next = -1;
                                        ctl->zsctl_proc_open_next = 0;
                                        continue;
                                }
                        } else {
                                /*
                                 * Other accounting error.  Give up on
                                 * accounting.
                                 */
                                goto ea_err;
                        }
                }
                /* Skip if not a process group */
                if ((object.eo_catalog & EXT_TYPE_MASK) != EXT_GROUP ||
                    (object.eo_catalog & EXD_DATA_MASK) != EXD_GROUP_PROC) {
                        (void) ea_free_item(&object, EUP_ALLOC);
                        continue;
                }

                /* The process group entry should be complete */
                while (numfound < 9) {
                        bzero(&pobject, sizeof (pobject));
                        ret = ea_get_object(&ctl->zsctl_proc_eaf,
                            &pobject);
                        if (ret < 0) {
                                (void) ea_free_item(&object, EUP_ALLOC);
                                zsd_warn(
                                    "unable to get process accounting data");
                                goto ea_err;
                        }
                        /* Next entries should be process data */
                        if ((pobject.eo_catalog & EXT_TYPE_MASK) ==
                            EXT_GROUP) {
                                (void) ea_free_item(&object, EUP_ALLOC);
                                (void) ea_free_item(&pobject, EUP_ALLOC);
                                zsd_warn(
                                    "process data of wrong type");
                                goto ea_err;
                        }
                        switch (pobject.eo_catalog & EXD_DATA_MASK) {
                        case EXD_PROC_PID:
                                pid = pobject.eo_item.ei_uint32;
                                proc = &(ctl->zsctl_proc_array[pid]);
                                /*
                                 * This process should not be currently in
                                 * the list of processes to process.
                                 */
                                assert(!list_link_active(&proc->zspr_next));
                                numfound++;
                                break;
                        case EXD_PROC_ANCPID:
                                ppid = pobject.eo_item.ei_uint32;
                                pproc = &(ctl->zsctl_proc_array[ppid]);
                                numfound++;
                                break;
                        case EXD_PROC_ZONENAME:
                                zone = zsd_lookup_zone(ctl,
                                    pobject.eo_item.ei_string, -1);
                                numfound++;
                                break;
                        case EXD_PROC_CPU_USER_SEC:
                                user.tv_sec =
                                    pobject.eo_item.ei_uint64;
                                numfound++;
                                break;
                        case EXD_PROC_CPU_USER_NSEC:
                                user.tv_nsec =
                                    pobject.eo_item.ei_uint64;
                                numfound++;
                                break;
                        case EXD_PROC_CPU_SYS_SEC:
                                sys.tv_sec =
                                    pobject.eo_item.ei_uint64;
                                numfound++;
                                break;
                        case EXD_PROC_CPU_SYS_NSEC:
                                sys.tv_nsec =
                                    pobject.eo_item.ei_uint64;
                                numfound++;
                                break;
                        case EXD_PROC_FINISH_SEC:
                                finish.tv_sec =
                                    pobject.eo_item.ei_uint64;
                                numfound++;
                                break;
                        case EXD_PROC_FINISH_NSEC:
                                finish.tv_nsec =
                                    pobject.eo_item.ei_uint64;
                                numfound++;
                                break;
                        }
                        (void) ea_free_item(&pobject, EUP_ALLOC);
                }
                (void) ea_free_item(&object, EUP_ALLOC);
                if (numfound != 9) {
                        zsd_warn(gettext(
                            "Malformed process accounting entry found"));
                        goto proc_done;
                }

                if (finish.tv_sec > interval_end.tv_sec ||
                    (finish.tv_sec == interval_end.tv_sec &&
                    finish.tv_nsec > (interval_end.tv_usec * 1000)))
                        hrtime_expired = B_TRUE;

                /*
                 * Try to identify the zone and pset to which this
                 * exited process belongs.
                 */
                if (zone == NULL)
                        goto proc_done;

                /* Save proc info */
                proc->zspr_ppid = ppid;
                proc->zspr_zoneid = zone->zsz_id;

                prev_psetid = ZS_PSET_ERROR;
                sched = 0;

                /*
                 * The following tries to deduce the processes pset.
                 *
                 * First choose pset and sched using cached value from the
                 * most recent time the process has been seen.
                 *
                 * pset and sched can change across zone_enter, so make sure
                 * most recent sighting of this process was in the same
                 * zone before using most recent known value.
                 *
                 * If there is no known value, use value of processes
                 * parent.  If parent is unknown, walk parents until a known
                 * parent is found.
                 *
                 * If no parent in the zone is found, use the zone's default
                 * pset and scheduling class.
                 */
                if (proc->zspr_psetid != ZS_PSET_ERROR) {
                        prev_psetid = proc->zspr_psetid;
                        pset = zsd_lookup_pset_byid(ctl, prev_psetid);
                        sched = proc->zspr_sched;
                } else if (pproc->zspr_zoneid == zone->zsz_id &&
                    pproc->zspr_psetid != ZS_PSET_ERROR) {
                        prev_psetid = pproc->zspr_psetid;
                        pset = zsd_lookup_pset_byid(ctl, prev_psetid);
                        sched = pproc->zspr_sched;
                }

                if (pset == NULL) {
                        /*
                         * Process or processes parent has never been seen.
                         * Save to deduce a known parent later.
                         */
                        proc_usage = sys;
                        TIMESTRUC_ADD_TIMESTRUC(proc_usage, user);
                        TIMESTRUC_DELTA(delta, proc_usage,
                            proc->zspr_usage);
                        proc->zspr_usage = delta;
                        list_insert_tail(&plist, proc);
                        continue;
                }

                /* Add the zone's usage to the pset */
                usage = zsd_lookup_insert_usage(ctl, pset, zone);
                if (usage == NULL)
                        goto proc_done;

                zsd_mark_pset_usage_found(usage, sched);

                /* compute the usage to add for the exited proc */
                proc_usage = sys;
                TIMESTRUC_ADD_TIMESTRUC(proc_usage, user);
                TIMESTRUC_DELTA(delta, proc_usage,
                    proc->zspr_usage);

                zsd_add_usage(ctl, usage, &delta);
proc_done:
                zsd_flush_proc_info(proc);

                if (hrtime_expired == B_TRUE)
                        break;
        }
        /*
         * close next accounting file.
         */
        if (ctl->zsctl_proc_open_next) {
                (void) ea_close(
                    &ctl->zsctl_proc_eaf_next);
                ctl->zsctl_proc_open_next = 0;
                ctl->zsctl_proc_fd_next = -1;
        }

        /* For the remaining processes, use pset and sched of a known parent */
        proc = list_head(&plist);
        while (proc != NULL) {
                next = proc;
                for (;;) {
                        if (next->zspr_ppid == 0 || next->zspr_ppid == -1) {
                                /*
                                 * Kernel process, or parent is unknown, skip
                                 * process, remove from process list.
                                 */
                                tmp = proc;
                                proc = list_next(&plist, proc);
                                list_link_init(&tmp->zspr_next);
                                break;
                        }
                        pproc = &(ctl->zsctl_proc_array[next->zspr_ppid]);
                        if (pproc->zspr_zoneid != proc->zspr_zoneid) {
                                /*
                                 * Parent in different zone.  Save process and
                                 * use zone's default pset and sched below
                                 */
                                tmp = proc;
                                proc = list_next(&plist, proc);
                                list_remove(&plist, tmp);
                                list_insert_tail(&pplist, tmp);
                                break;
                        }
                        /* Parent has unknown pset, Search parent's parent  */
                        if (pproc->zspr_psetid == ZS_PSET_ERROR) {
                                next = pproc;
                                continue;
                        }
                        /* Found parent with known pset.  Use its info */
                        proc->zspr_psetid = pproc->zspr_psetid;
                        proc->zspr_sched = pproc->zspr_sched;
                        next->zspr_psetid = pproc->zspr_psetid;
                        next->zspr_sched = pproc->zspr_sched;
                        zone = zsd_lookup_zone_byid(ctl,
                            proc->zspr_zoneid);
                        if (zone == NULL) {
                                tmp = proc;
                                proc = list_next(&plist, proc);
                                list_remove(&plist, tmp);
                                list_link_init(&tmp->zspr_next);
                                break;
                        }
                        pset = zsd_lookup_pset_byid(ctl,
                            proc->zspr_psetid);
                        if (pset == NULL) {
                                tmp = proc;
                                proc = list_next(&plist, proc);
                                list_remove(&plist, tmp);
                                list_link_init(&tmp->zspr_next);
                                break;
                        }
                        /* Add the zone's usage to the pset */
                        usage = zsd_lookup_insert_usage(ctl, pset, zone);
                        if (usage == NULL) {
                                tmp = proc;
                                proc = list_next(&plist, proc);
                                list_remove(&plist, tmp);
                                list_link_init(&tmp->zspr_next);
                                break;
                        }
                        zsd_mark_pset_usage_found(usage, proc->zspr_sched);
                        zsd_add_usage(ctl, usage, &proc->zspr_usage);
                        zsd_flush_proc_info(proc);
                        tmp = proc;
                        proc = list_next(&plist, proc);
                        list_remove(&plist, tmp);
                        list_link_init(&tmp->zspr_next);
                        break;
                }
        }
        /*
         * Process has never been seen.  Using zone info to
         * determine pset and scheduling class.
         */
        proc = list_head(&pplist);
        while (proc != NULL) {

                zone = zsd_lookup_zone_byid(ctl, proc->zspr_zoneid);
                if (zone == NULL)
                        goto next;
                if (zone->zsz_psetid != ZS_PSET_ERROR &&
                    zone->zsz_psetid != ZS_PSET_MULTI) {
                        prev_psetid = zone->zsz_psetid;
                        pset = zsd_lookup_pset_byid(ctl, prev_psetid);
                } else {
                        pset = zsd_lookup_pset(ctl, zone->zsz_pset, -1);
                        if (pset != NULL)
                                prev_psetid = pset->zsp_id;
                }
                if (pset == NULL)
                        goto next;

                sched = zone->zsz_scheds;
                /*
                 * Ignore FX high scheduling class if it is not the
                 * only scheduling class in the zone.
                 */
                if (sched != ZS_SCHED_FX_60)
                        sched &= (~ZS_SCHED_FX_60);
                /*
                 * If more than one scheduling class has been found
                 * in the zone, use zone's default scheduling class for
                 * this process.
                 */
                if ((sched & (sched - 1)) != 0)
                        sched = zone->zsz_default_sched;

                /* Add the zone's usage to the pset */
                usage = zsd_lookup_insert_usage(ctl, pset, zone);
                if (usage == NULL)
                        goto next;

                zsd_mark_pset_usage_found(usage, sched);
                zsd_add_usage(ctl, usage, &proc->zspr_usage);
next:
                tmp = proc;
                proc = list_next(&pplist, proc);
                zsd_flush_proc_info(tmp);
                list_link_init(&tmp->zspr_next);
        }
        return;
ea_err:
        /*
         * Close the next accounting file if we have not transitioned to it
         * yet.
         */
        if (ctl->zsctl_proc_open_next) {
                (void) ea_close(&ctl->zsctl_proc_eaf_next);
                ctl->zsctl_proc_open_next = 0;
                ctl->zsctl_proc_fd_next = -1;
        }
}

/*
 * getvmusage(2) uses size_t's in the passwd data structure, which differ
 * in size for 32bit and 64 bit kernels.  Since this is a contracted interface,
 * and zonestatd does not necessarily match the kernel's bitness, marshal
 * results appropriately.
 */
static int
zsd_getvmusage(zsd_ctl_t *ctl, uint_t flags, time_t age, zsd_vmusage64_t *buf,
    uint64_t *nres)
{
        zsd_vmusage32_t *vmu32;
        zsd_vmusage64_t *vmu64;
        uint32_t nres32;
        int i;
        int ret;

        if (ctl->zsctl_kern_bits == 32)  {
                nres32 = *nres;
                ret = syscall(SYS_rusagesys, _RUSAGESYS_GETVMUSAGE,
                    flags, age, (uintptr_t)buf, (uintptr_t)&nres32);
                *nres = nres32;
                if (ret == 0 && buf != NULL) {
                        /*
                         * An array of vmusage32_t's has been returned.
                         * Convert it to an array of vmusage64_t's.
                         */
                        vmu32 = (zsd_vmusage32_t *)buf;
                        vmu64 = (zsd_vmusage64_t *)buf;
                        for (i = nres32 - 1; i >= 0; i--) {

                                vmu64[i].vmu_zoneid = vmu32[i].vmu_zoneid;
                                vmu64[i].vmu_type = vmu32[i].vmu_type;
                                vmu64[i].vmu_type = vmu32[i].vmu_type;
                                vmu64[i].vmu_rss_all = vmu32[i].vmu_rss_all;
                                vmu64[i].vmu_rss_private =
                                    vmu32[i].vmu_rss_private;
                                vmu64[i].vmu_rss_shared =
                                    vmu32[i].vmu_rss_shared;
                                vmu64[i].vmu_swap_all = vmu32[i].vmu_swap_all;
                                vmu64[i].vmu_swap_private =
                                    vmu32[i].vmu_swap_private;
                                vmu64[i].vmu_swap_shared =
                                    vmu32[i].vmu_swap_shared;
                        }
                }
                return (ret);
        } else {
                /*
                 * kernel is 64 bit, so use 64 bit structures as zonestat
                 * expects.
                 */
                return (syscall(SYS_rusagesys, _RUSAGESYS_GETVMUSAGE,
                    flags, age, (uintptr_t)buf, (uintptr_t)nres));

        }
}

/*
 * Update the current physical, virtual, and locked memory usage of the
 * running zones.
 */
static void
zsd_refresh_memory(zsd_ctl_t *ctl, boolean_t init)
{

        uint64_t phys_total;
        uint64_t phys_used;
        uint64_t phys_zones;
        uint64_t phys_zones_overcount;
        uint64_t phys_zones_extra;
        uint64_t phys_zones_credit;

        uint64_t vm_free;
        uint64_t vm_used;

        uint64_t disk_swap_total;
        uint64_t disk_swap_used;        /* disk swap with contents */

        uint64_t physmem;
        uint64_t pp_kernel;
        uint64_t arc_size = 0;
        struct anoninfo ani;

        int num_swap_devices;
        struct swaptable *swt;
        struct swapent *swent;
        size_t swt_size;
        char *path;

        zsd_vmusage64_t *vmusage;
        uint64_t num_vmusage;

        int i, ret;

        zsd_system_t *sys;
        zsd_zone_t *zone;
        int vmu_nzones;

        kstat_t *kstat;
        char kstat_name[KSTAT_STRLEN];
        kstat_named_t *knp;
        kid_t kid;

        if (init)
                return;

        sys = ctl->zsctl_system;

        /* interrogate swap devices to find the amount of disk swap */
disk_swap_again:
        num_swap_devices = swapctl(SC_GETNSWP, NULL);

        if (num_swap_devices == 0) {
                sys->zss_swap_total = disk_swap_total = 0;
                sys->zss_swap_used = disk_swap_used = 0;
                /* No disk swap */
                goto disk_swap_done;
        }
        /* see if swap table needs to be larger */
        if (num_swap_devices > ctl->zsctl_swap_cache_num) {
                swt_size = sizeof (int) +
                    (num_swap_devices * sizeof (struct swapent)) +
                    (num_swap_devices * MAXPATHLEN);
                free(ctl->zsctl_swap_cache);

                swt = (struct swaptable *)malloc(swt_size);
                if (swt == NULL) {
                        /*
                         * Could not allocate to get list of swap devices.
                         * Just use data from the most recent read, which will
                         * be zero if this is the first read.
                         */
                        zsd_warn(gettext("Unable to allocate to determine "
                            "virtual memory"));
                        disk_swap_total = sys->zss_swap_total;
                        disk_swap_used = sys->zss_swap_used;
                        goto disk_swap_done;
                }
                swent = swt->swt_ent;
                path = (char *)swt + (sizeof (int) +
                    num_swap_devices * sizeof (swapent_t));
                for (i = 0; i < num_swap_devices; i++, swent++) {
                        swent->ste_path = path;
                        path += MAXPATHLEN;
                }
                swt->swt_n = num_swap_devices;
                ctl->zsctl_swap_cache = swt;
                ctl->zsctl_swap_cache_size = swt_size;
                ctl->zsctl_swap_cache_num = num_swap_devices;
        }
        num_swap_devices = swapctl(SC_LIST, ctl->zsctl_swap_cache);
        if (num_swap_devices < 0) {
                /* More swap devices have arrived */
                if (errno == ENOMEM)
                        goto disk_swap_again;

                zsd_warn(gettext("Unable to determine disk swap devices"));
                /* Unexpected error.  Use existing data */
                disk_swap_total = sys->zss_swap_total;
                disk_swap_used = sys->zss_swap_used;
                goto disk_swap_done;
        }

        /* add up the disk swap */
        disk_swap_total = 0;
        disk_swap_used = 0;
        swent = ctl->zsctl_swap_cache->swt_ent;
        for (i = 0; i < num_swap_devices; i++, swent++) {
                disk_swap_total += swent->ste_pages;
                disk_swap_used += (swent->ste_pages - swent->ste_free);
        }
        disk_swap_total *= ctl->zsctl_pagesize;
        disk_swap_used *= ctl->zsctl_pagesize;

        sys->zss_swap_total = disk_swap_total;
        sys->zss_swap_used = disk_swap_used;

disk_swap_done:

        /* get system pages kstat */
        kid = -1;
        kstat = kstat_lookup(ctl->zsctl_kstat_ctl, "unix", 0, "system_pages");
        if (kstat == NULL)
                zsd_warn(gettext("Unable to lookup system pages kstat"));
        else
                kid = kstat_read(ctl->zsctl_kstat_ctl, kstat, NULL);

        if (kid == -1) {
                zsd_warn(gettext("Unable to read system pages kstat"));
                return;
        } else {
                knp = kstat_data_lookup(kstat, "physmem");
                if (knp == NULL) {
                        zsd_warn(gettext("Unable to read physmem"));
                } else {
                        if (knp->data_type == KSTAT_DATA_UINT64)
                                physmem = knp->value.ui64;
                        else if (knp->data_type == KSTAT_DATA_UINT32)
                                physmem = knp->value.ui32;
                        else
                                return;
                }
                knp = kstat_data_lookup(kstat, "pp_kernel");
                if (knp == NULL) {
                        zsd_warn(gettext("Unable to read pp_kernel"));
                } else {
                        if (knp->data_type == KSTAT_DATA_UINT64)
                                pp_kernel = knp->value.ui64;
                        else if (knp->data_type == KSTAT_DATA_UINT32)
                                pp_kernel = knp->value.ui32;
                        else
                                return;
                }
        }
        physmem *= ctl->zsctl_pagesize;
        pp_kernel *= ctl->zsctl_pagesize;

        /* get the zfs arc size if available */
        arc_size = 0;
        kid = -1;
        kstat = kstat_lookup(ctl->zsctl_kstat_ctl, "zfs", 0, "arcstats");
        if (kstat != NULL)
                kid = kstat_read(ctl->zsctl_kstat_ctl, kstat, NULL);
        if (kid != -1) {
                knp = kstat_data_lookup(kstat, "size");
                if (knp != NULL)
                        if (knp->data_type == KSTAT_DATA_UINT64)
                                arc_size = knp->value.ui64;
        }

        /* Try to get swap information */
        if (swapctl(SC_AINFO, &ani) < 0) {
                zsd_warn(gettext("Unable to get swap info"));
                return;
        }

vmusage_again:
        /* getvmusage to get physical memory usage */
        vmusage = ctl->zsctl_vmusage_cache;
        num_vmusage = ctl->zsctl_vmusage_cache_num;

        ret = zsd_getvmusage(ctl, VMUSAGE_SYSTEM | VMUSAGE_ALL_ZONES, 0,
            vmusage, &num_vmusage);

        if (ret != 0) {
                /* Unexpected error.  Use existing data */
                if (errno != EOVERFLOW) {
                        zsd_warn(gettext(
                            "Unable to read physical memory usage"));
                        phys_zones = sys->zss_ram_zones;
                        goto vmusage_done;
                }
        }
        /* vmusage results cache too small */
        if (num_vmusage > ctl->zsctl_vmusage_cache_num) {

                size_t size = sizeof (zsd_vmusage64_t) * num_vmusage;

                free(ctl->zsctl_vmusage_cache);
                vmusage = (zsd_vmusage64_t *)malloc(size);
                if (vmusage == NULL) {
                        zsd_warn(gettext("Unable to alloc to determine "
                            "physical memory usage"));
                        phys_zones = sys->zss_ram_zones;
                        goto vmusage_done;
                }
                ctl->zsctl_vmusage_cache = vmusage;
                ctl->zsctl_vmusage_cache_num = num_vmusage;
                goto vmusage_again;
        }

        phys_zones_overcount = 0;
        vmu_nzones = 0;
        for (i = 0; i < num_vmusage; i++) {
                switch (vmusage[i].vmu_type) {
                case VMUSAGE_SYSTEM:
                        /* total pages backing user process mappings */
                        phys_zones = sys->zss_ram_zones =
                            vmusage[i].vmu_rss_all;
                        break;
                case VMUSAGE_ZONE:
                        vmu_nzones++;
                        phys_zones_overcount += vmusage[i].vmu_rss_all;
                        zone = zsd_lookup_zone_byid(ctl, vmusage[i].vmu_id);
                        if (zone != NULL)
                                zone->zsz_usage_ram = vmusage[i].vmu_rss_all;
                        break;
                default:
                        break;
                }
        }
        /*
         * Figure how much memory was double counted due to text sharing
         * between zones.  Credit this back so that the sum of the zones
         * equals the total zone ram usage;
         */
        phys_zones_extra = phys_zones_overcount - phys_zones;
        phys_zones_credit = phys_zones_extra / vmu_nzones;

vmusage_done:

        /* walk the zones to get swap and locked kstats.  Fetch ram cap. */
        sys->zss_locked_zones = 0;
        sys->zss_vm_zones = 0;
        for (zone = list_head(&ctl->zsctl_zones); zone != NULL;
            zone = list_next(&ctl->zsctl_zones, zone)) {

                /* If zone halted during interval, show memory usage as none */
                if (zone->zsz_active == B_FALSE ||
                    zone->zsz_deleted == B_TRUE) {
                        zone->zsz_usage_ram = 0;
                        zone->zsz_usage_vm = 0;
                        zone->zsz_usage_locked = 0;
                        continue;
                }

                if (phys_zones_credit > 0) {
                        if (zone->zsz_usage_ram > phys_zones_credit) {
                                zone->zsz_usage_ram -= phys_zones_credit;
                        }
                }
                /*
                 * Get zone's swap usage.  Since zone could have halted,
                 * treats as zero if cannot read
                 */
                zone->zsz_usage_vm = 0;
                (void) snprintf(kstat_name, sizeof (kstat_name),
                    "swapresv_zone_%d", zone->zsz_id);
                kid = -1;
                kstat = kstat_lookup(ctl->zsctl_kstat_ctl, "caps",
                    zone->zsz_id, kstat_name);
                if (kstat != NULL)
                        kid = kstat_read(ctl->zsctl_kstat_ctl, kstat, NULL);
                if (kid != -1) {
                        knp = kstat_data_lookup(kstat, "usage");
                        if (knp != NULL &&
                            knp->data_type == KSTAT_DATA_UINT64) {
                                zone->zsz_usage_vm = knp->value.ui64;
                                sys->zss_vm_zones += knp->value.ui64;
                        }
                }
                /*
                 * Get zone's locked usage.  Since zone could have halted,
                 * treats as zero if cannot read
                 */
                zone->zsz_usage_locked = 0;
                (void) snprintf(kstat_name, sizeof (kstat_name),
                    "lockedmem_zone_%d", zone->zsz_id);
                kid = -1;
                kstat = kstat_lookup(ctl->zsctl_kstat_ctl, "caps",
                    zone->zsz_id, kstat_name);
                if (kstat != NULL)
                        kid = kstat_read(ctl->zsctl_kstat_ctl, kstat, NULL);
                if (kid != -1) {
                        knp = kstat_data_lookup(kstat, "usage");
                        if (knp != NULL &&
                            knp->data_type == KSTAT_DATA_UINT64) {
                                zone->zsz_usage_locked = knp->value.ui64;
                                /*
                                 * Since locked memory accounting for zones
                                 * can double count ddi locked memory, cap each
                                 * zone's locked usage at its ram usage.
                                 */
                                if (zone->zsz_usage_locked >
                                    zone->zsz_usage_ram)
                                        zone->zsz_usage_locked =
                                            zone->zsz_usage_ram;
                                sys->zss_locked_zones +=
                                    zone->zsz_usage_locked;
                        }
                }
        }

        phys_total =
            sysconf(_SC_PHYS_PAGES) * ctl->zsctl_pagesize;

        phys_used = (sysconf(_SC_PHYS_PAGES) - sysconf(_SC_AVPHYS_PAGES))
            * ctl->zsctl_pagesize;

        /* Compute remaining statistics */
        sys->zss_ram_total = phys_total;
        sys->zss_ram_zones = phys_zones;
        sys->zss_ram_kern = phys_used - phys_zones - arc_size;

        /*
         * The total for kernel locked memory should include
         * segkp locked pages, but oh well.  The arc size is subtracted,
         * as that physical memory is reclaimable.
         */
        sys->zss_locked_kern = pp_kernel - arc_size;
        /* Add memory used by kernel startup and obp to kernel locked */
        if ((phys_total - physmem) > 0)
                sys->zss_locked_kern += phys_total - physmem;

        /*
         * Add in the portion of (RAM+DISK) that is not available as swap,
         * and consider it swap used by the kernel.
         */
        sys->zss_vm_total = phys_total + disk_swap_total;
        vm_free = (ani.ani_max - ani.ani_resv) * ctl->zsctl_pagesize;
        vm_used = sys->zss_vm_total - vm_free;
        sys->zss_vm_kern = vm_used - sys->zss_vm_zones - arc_size;
}

/*
 * Charge each cpu's usage to its processor sets.  Also add the cpu's total
 * time to each zone using the processor set.  This tracks the maximum
 * amount of cpu time that a zone could have used.
 */
static void
zsd_refresh_cpu_stats(zsd_ctl_t *ctl, boolean_t init)
{
        zsd_system_t *sys;
        zsd_zone_t *zone;
        zsd_pset_usage_t *usage;
        zsd_cpu_t *cpu;
        zsd_cpu_t *cpu_next;
        zsd_pset_t *pset;
        timestruc_t ts;
        uint64_t hrtime;
        timestruc_t delta;

        /* Update the per-cpu kstat data */
        cpu_next = list_head(&ctl->zsctl_cpus);
        while (cpu_next != NULL) {
                cpu = cpu_next;
                cpu_next = list_next(&ctl->zsctl_cpus, cpu);
                zsd_update_cpu_stats(ctl, cpu);
        }
        /* Update the elapsed real time */
        hrtime = gethrtime();
        if (init) {
                /* first time around, store hrtime for future comparision */
                ctl->zsctl_hrtime = hrtime;
                ctl->zsctl_hrtime_prev = hrtime;

        } else {
                /* Compute increase in hrtime since the most recent read */
                ctl->zsctl_hrtime_prev = ctl->zsctl_hrtime;
                ctl->zsctl_hrtime = hrtime;
                if ((hrtime = hrtime - ctl->zsctl_hrtime_prev) > 0)
                        TIMESTRUC_ADD_NANOSEC(ctl->zsctl_hrtime_total, hrtime);
        }

        /* On initialization, all psets have zero time  */
        if (init)
                return;

        for (pset = list_head(&ctl->zsctl_psets); pset != NULL;
            pset = list_next(&ctl->zsctl_psets, pset)) {

                if (pset->zsp_active == B_FALSE) {
                        zsd_warn(gettext("Internal error,inactive pset found"));
                        continue;
                }

                /* sum total used time for pset */
                ts.tv_sec = 0;
                ts.tv_nsec = 0;
                TIMESTRUC_ADD_TIMESTRUC(ts, pset->zsp_intr);
                TIMESTRUC_ADD_TIMESTRUC(ts, pset->zsp_kern);
                TIMESTRUC_ADD_TIMESTRUC(ts, pset->zsp_user);
                /* kernel time in pset is total time minus zone time */
                TIMESTRUC_DELTA(pset->zsp_usage_kern, ts,
                    pset->zsp_usage_zones);
                if (pset->zsp_usage_kern.tv_sec < 0 ||
                    pset->zsp_usage_kern.tv_nsec < 0) {
                        pset->zsp_usage_kern.tv_sec = 0;
                        pset->zsp_usage_kern.tv_nsec = 0;
                }
                /* Total pset elapsed time is used time plus idle time */
                TIMESTRUC_ADD_TIMESTRUC(ts, pset->zsp_idle);

                TIMESTRUC_DELTA(delta, ts, pset->zsp_total_time);

                for (usage = list_head(&pset->zsp_usage_list); usage != NULL;
                    usage = list_next(&pset->zsp_usage_list, usage)) {

                        zone = usage->zsu_zone;
                        if (usage->zsu_cpu_shares != ZS_LIMIT_NONE &&
                            usage->zsu_cpu_shares != ZS_SHARES_UNLIMITED &&
                            usage->zsu_cpu_shares != 0) {
                                /*
                                 * Figure out how many nanoseconds of share time
                                 * to give to the zone
                                 */
                                hrtime = delta.tv_sec;
                                hrtime *= NANOSEC;
                                hrtime += delta.tv_nsec;
                                hrtime *= usage->zsu_cpu_shares;
                                hrtime /= pset->zsp_cpu_shares;
                                TIMESTRUC_ADD_NANOSEC(zone->zsz_share_time,
                                    hrtime);
                        }
                        /* Add pset time to each zone using pset */
                        TIMESTRUC_ADD_TIMESTRUC(zone->zsz_pset_time, delta);

                        zone->zsz_cpus_online += pset->zsp_online;
                }
                pset->zsp_total_time = ts;
        }

        for (zone = list_head(&ctl->zsctl_zones); zone != NULL;
            zone = list_next(&ctl->zsctl_zones, zone)) {

                /* update cpu cap tracking if the zone has a cpu cap */
                if (zone->zsz_cpu_cap != ZS_LIMIT_NONE) {
                        uint64_t elapsed;

                        elapsed = ctl->zsctl_hrtime - ctl->zsctl_hrtime_prev;
                        elapsed *= zone->zsz_cpu_cap;
                        elapsed = elapsed / 100;
                        TIMESTRUC_ADD_NANOSEC(zone->zsz_cap_time, elapsed);
                }
        }
        sys = ctl->zsctl_system;
        ts.tv_sec = 0;
        ts.tv_nsec = 0;
        TIMESTRUC_ADD_TIMESTRUC(ts, sys->zss_intr);
        TIMESTRUC_ADD_TIMESTRUC(ts, sys->zss_kern);
        TIMESTRUC_ADD_TIMESTRUC(ts, sys->zss_user);

        /* kernel time in pset is total time minus zone time */
        TIMESTRUC_DELTA(sys->zss_cpu_usage_kern, ts,
            sys->zss_cpu_usage_zones);
        if (sys->zss_cpu_usage_kern.tv_sec < 0 ||
            sys->zss_cpu_usage_kern.tv_nsec < 0) {
                sys->zss_cpu_usage_kern.tv_sec = 0;
                sys->zss_cpu_usage_kern.tv_nsec = 0;
        }
        /* Total pset elapsed time is used time plus idle time */
        TIMESTRUC_ADD_TIMESTRUC(ts, sys->zss_idle);
        sys->zss_cpu_total_time = ts;
}

/*
 * Saves current usage data to a cache that is read by libzonestat when
 * calling zs_usage_read().
 *
 * All pointers in the cached data structure are set to NULL.  When
 * libzonestat reads the cached data, it will set the pointers relative to
 * its address space.
 */
static void
zsd_usage_cache_update(zsd_ctl_t *ctl)
{
        zs_usage_cache_t *cache;
        zs_usage_cache_t *old;
        zs_usage_t *usage;

        zs_system_t *sys;
        zsd_system_t *dsys;
        zs_zone_t *zone = NULL;
        zsd_zone_t *dzone;
        zs_pset_t *pset = NULL;
        zsd_pset_t *dpset;
        zs_pset_zone_t *pusage;
        zsd_pset_usage_t *dpusage;

        char *next;
        uint_t size, i, j;

        size =
            sizeof (zs_usage_cache_t) +
            sizeof (zs_usage_t) +
            sizeof (zs_system_t) +
            sizeof (zs_zone_t) * ctl->zsctl_nzones +
            sizeof (zs_pset_t) *  ctl->zsctl_npsets +
            sizeof (zs_pset_zone_t) * ctl->zsctl_npset_usages;

        cache = (zs_usage_cache_t *)malloc(size);
        if (cache == NULL) {
                zsd_warn(gettext("Unable to allocate usage cache\n"));
                return;
        }

        next = (char *)cache;
        cache->zsuc_size = size - sizeof (zs_usage_cache_t);
        next += sizeof (zs_usage_cache_t);

        /* LINTED */
        usage = cache->zsuc_usage = (zs_usage_t *)next;
        next += sizeof (zs_usage_t);
        usage->zsu_start = g_start;
        usage->zsu_hrstart = g_hrstart;
        usage->zsu_time = g_now;
        usage->zsu_hrtime = g_hrnow;
        usage->zsu_nzones = ctl->zsctl_nzones;
        usage->zsu_npsets = ctl->zsctl_npsets;
        usage->zsu_system = NULL;

        /* LINTED */
        sys = (zs_system_t *)next;
        next += sizeof (zs_system_t);
        dsys = ctl->zsctl_system;
        sys->zss_ram_total = dsys->zss_ram_total;
        sys->zss_ram_kern = dsys->zss_ram_kern;
        sys->zss_ram_zones = dsys->zss_ram_zones;
        sys->zss_locked_kern = dsys->zss_locked_kern;
        sys->zss_locked_zones = dsys->zss_locked_zones;
        sys->zss_vm_total = dsys->zss_vm_total;
        sys->zss_vm_kern = dsys->zss_vm_kern;
        sys->zss_vm_zones = dsys->zss_vm_zones;
        sys->zss_swap_total = dsys->zss_swap_total;
        sys->zss_swap_used = dsys->zss_swap_used;
        sys->zss_ncpus = dsys->zss_ncpus;
        sys->zss_ncpus_online = dsys->zss_ncpus_online;

        sys->zss_processes_max = dsys->zss_maxpid;
        sys->zss_lwps_max = dsys->zss_lwps_max;
        sys->zss_shm_max = dsys->zss_shm_max;
        sys->zss_shmids_max = dsys->zss_shmids_max;
        sys->zss_semids_max = dsys->zss_semids_max;
        sys->zss_msgids_max = dsys->zss_msgids_max;
        sys->zss_lofi_max = dsys->zss_lofi_max;

        sys->zss_processes = dsys->zss_processes;
        sys->zss_lwps = dsys->zss_lwps;
        sys->zss_shm = dsys->zss_shm;
        sys->zss_shmids = dsys->zss_shmids;
        sys->zss_semids = dsys->zss_semids;
        sys->zss_msgids = dsys->zss_msgids;
        sys->zss_lofi = dsys->zss_lofi;

        sys->zss_cpu_total_time = dsys->zss_cpu_total_time;
        sys->zss_cpu_usage_zones = dsys->zss_cpu_usage_zones;
        sys->zss_cpu_usage_kern = dsys->zss_cpu_usage_kern;

        for (i = 0, dzone = list_head(&ctl->zsctl_zones);
            i < ctl->zsctl_nzones;
            i++, dzone = list_next(&ctl->zsctl_zones, dzone)) {
                /* LINTED */
                zone = (zs_zone_t *)next;
                next += sizeof (zs_zone_t);
                list_link_init(&zone->zsz_next);
                zone->zsz_system = NULL;

                (void) strlcpy(zone->zsz_name, dzone->zsz_name,
                    sizeof (zone->zsz_name));
                (void) strlcpy(zone->zsz_pool, dzone->zsz_pool,
                    sizeof (zone->zsz_pool));
                (void) strlcpy(zone->zsz_pset, dzone->zsz_pset,
                    sizeof (zone->zsz_pset));
                zone->zsz_id = dzone->zsz_id;
                zone->zsz_cputype = dzone->zsz_cputype;
                zone->zsz_iptype = dzone->zsz_iptype;
                zone->zsz_start = dzone->zsz_start;
                zone->zsz_hrstart = dzone->zsz_hrstart;
                zone->zsz_scheds = dzone->zsz_scheds;
                zone->zsz_cpu_shares = dzone->zsz_cpu_shares;
                zone->zsz_cpu_cap = dzone->zsz_cpu_cap;
                zone->zsz_ram_cap = dzone->zsz_ram_cap;
                zone->zsz_vm_cap = dzone->zsz_vm_cap;
                zone->zsz_locked_cap = dzone->zsz_locked_cap;
                zone->zsz_cpu_usage = dzone->zsz_cpu_usage;
                zone->zsz_cpus_online = dzone->zsz_cpus_online;
                zone->zsz_pset_time = dzone->zsz_pset_time;
                zone->zsz_cap_time = dzone->zsz_cap_time;
                zone->zsz_share_time = dzone->zsz_share_time;
                zone->zsz_usage_ram = dzone->zsz_usage_ram;
                zone->zsz_usage_locked = dzone->zsz_usage_locked;
                zone->zsz_usage_vm = dzone->zsz_usage_vm;

                zone->zsz_processes_cap = dzone->zsz_processes_cap;
                zone->zsz_lwps_cap = dzone->zsz_lwps_cap;
                zone->zsz_shm_cap = dzone->zsz_shm_cap;
                zone->zsz_shmids_cap = dzone->zsz_shmids_cap;
                zone->zsz_semids_cap = dzone->zsz_semids_cap;
                zone->zsz_msgids_cap = dzone->zsz_msgids_cap;
                zone->zsz_lofi_cap = dzone->zsz_lofi_cap;

                zone->zsz_processes = dzone->zsz_processes;
                zone->zsz_lwps = dzone->zsz_lwps;
                zone->zsz_shm = dzone->zsz_shm;
                zone->zsz_shmids = dzone->zsz_shmids;
                zone->zsz_semids = dzone->zsz_semids;
                zone->zsz_msgids = dzone->zsz_msgids;
                zone->zsz_lofi = dzone->zsz_lofi;
        }

        for (i = 0, dpset = list_head(&ctl->zsctl_psets);
            i < ctl->zsctl_npsets;
            i++, dpset = list_next(&ctl->zsctl_psets, dpset)) {
                /* LINTED */
                pset = (zs_pset_t *)next;
                next += sizeof (zs_pset_t);
                list_link_init(&pset->zsp_next);
                (void) strlcpy(pset->zsp_name, dpset->zsp_name,
                    sizeof (pset->zsp_name));
                pset->zsp_id = dpset->zsp_id;
                pset->zsp_cputype = dpset->zsp_cputype;
                pset->zsp_start = dpset->zsp_start;
                pset->zsp_hrstart = dpset->zsp_hrstart;
                pset->zsp_online = dpset->zsp_online;
                pset->zsp_size = dpset->zsp_size;
                pset->zsp_min = dpset->zsp_min;
                pset->zsp_max = dpset->zsp_max;
                pset->zsp_importance = dpset->zsp_importance;
                pset->zsp_scheds = dpset->zsp_scheds;
                pset->zsp_cpu_shares = dpset->zsp_cpu_shares;
                pset->zsp_total_time = dpset->zsp_total_time;
                pset->zsp_usage_kern = dpset->zsp_usage_kern;
                pset->zsp_usage_zones = dpset->zsp_usage_zones;
                pset->zsp_nusage = dpset->zsp_nusage;
                /* Add pset usages for pset */
                for (j = 0, dpusage = list_head(&dpset->zsp_usage_list);
                    j < dpset->zsp_nusage;
                    j++, dpusage = list_next(&dpset->zsp_usage_list, dpusage)) {
                        /* LINTED */
                        pusage = (zs_pset_zone_t *)next;
                        next += sizeof (zs_pset_zone_t);
                        /* pointers are computed by client */
                        pusage->zspz_pset = NULL;
                        pusage->zspz_zone = NULL;
                        list_link_init(&pusage->zspz_next);
                        pusage->zspz_zoneid = dpusage->zsu_zone->zsz_id;
                        pusage->zspz_start = dpusage->zsu_start;
                        pusage->zspz_hrstart = dpusage->zsu_hrstart;
                        pusage->zspz_hrstart = dpusage->zsu_hrstart;
                        pusage->zspz_cpu_shares = dpusage->zsu_cpu_shares;
                        pusage->zspz_scheds = dpusage->zsu_scheds;
                        pusage->zspz_cpu_usage = dpusage->zsu_cpu_usage;
                }
        }

        /* Update the current cache pointer */
        (void) mutex_lock(&g_usage_cache_lock);
                old = g_usage_cache;
                cache->zsuc_ref = 1;
                cache->zsuc_gen = g_gen_next;
                usage->zsu_gen = g_gen_next;
                usage->zsu_size = size;
                g_usage_cache = cache;
                if (old != NULL) {
                        old->zsuc_ref--;
                        if (old->zsuc_ref == 0)
                                free(old);
                }
                g_gen_next++;
        /* Wake up any clients that are waiting for this calculation */
        if (g_usage_cache_kickers > 0) {
                (void) cond_broadcast(&g_usage_cache_wait);
        }
        (void) mutex_unlock(&g_usage_cache_lock);
}

static zs_usage_cache_t *
zsd_usage_cache_hold_locked()
{
        zs_usage_cache_t *ret;

        ret = g_usage_cache;
        ret->zsuc_ref++;
        return (ret);
}

void
zsd_usage_cache_rele(zs_usage_cache_t *cache)
{
        (void) mutex_lock(&g_usage_cache_lock);
        cache->zsuc_ref--;
        if (cache->zsuc_ref == 0)
                free(cache);
        (void) mutex_unlock(&g_usage_cache_lock);
}

/* Close the handles held by zsd_open() */
void
zsd_close(zsd_ctl_t *ctl)
{
        zsd_zone_t *zone;
        zsd_pset_t *pset;
        zsd_pset_usage_t *usage;
        zsd_cpu_t *cpu;
        int id;

        if (ctl->zsctl_kstat_ctl) {
                (void) kstat_close(ctl->zsctl_kstat_ctl);
                ctl->zsctl_kstat_ctl = NULL;
        }
        if (ctl->zsctl_proc_open) {
                (void) ea_close(&ctl->zsctl_proc_eaf);
                ctl->zsctl_proc_open = 0;
                ctl->zsctl_proc_fd = -1;
        }
        if (ctl->zsctl_pool_conf) {
                if (ctl->zsctl_pool_status == POOL_ENABLED)
                        (void) pool_conf_close(ctl->zsctl_pool_conf);
                ctl->zsctl_pool_status = POOL_DISABLED;
        }

        while ((zone = list_head(&ctl->zsctl_zones)) != NULL) {
                list_remove(&ctl->zsctl_zones, zone);
                free(zone);
                ctl->zsctl_nzones--;
        }

        while ((pset = list_head(&ctl->zsctl_psets)) != NULL) {
                while ((usage = list_head(&pset->zsp_usage_list))
                    != NULL) {
                        list_remove(&pset->zsp_usage_list, usage);
                        ctl->zsctl_npset_usages--;
                        free(usage);
                }
                list_remove(&ctl->zsctl_psets, pset);
                free(pset);
                ctl->zsctl_npsets--;
        }

        /* Release all cpus being tracked */
        while (cpu = list_head(&ctl->zsctl_cpus)) {
                list_remove(&ctl->zsctl_cpus, cpu);
                id = cpu->zsc_id;
                bzero(cpu, sizeof (zsd_cpu_t));
                cpu->zsc_id = id;
                cpu->zsc_allocated = B_FALSE;
                cpu->zsc_psetid = ZS_PSET_ERROR;
                cpu->zsc_psetid_prev = ZS_PSET_ERROR;
        }

        assert(ctl->zsctl_npset_usages == 0);
        assert(ctl->zsctl_npsets == 0);
        assert(ctl->zsctl_nzones == 0);
        (void) zsd_disable_cpu_stats();
}


/*
 * Update the utilization data for all zones and processor sets.
 */
static int
zsd_read(zsd_ctl_t *ctl, boolean_t init, boolean_t do_memory)
{
        (void) kstat_chain_update(ctl->zsctl_kstat_ctl);
        (void) gettimeofday(&(ctl->zsctl_timeofday), NULL);

        zsd_refresh_system(ctl);

        /*
         * Memory calculation is expensive.  Only update it on sample
         * intervals.
         */
        if (do_memory == B_TRUE)
                zsd_refresh_memory(ctl, init);
        zsd_refresh_zones(ctl);
        zsd_refresh_psets(ctl);
        zsd_refresh_procs(ctl, init);
        zsd_refresh_cpu_stats(ctl, init);

        /*
         * Delete objects that no longer exist.
         * Pset usages must be deleted first as they point to zone and
         * pset objects.
         */
        zsd_mark_pset_usages_end(ctl);
        zsd_mark_psets_end(ctl);
        zsd_mark_cpus_end(ctl);
        zsd_mark_zones_end(ctl);

        /*
         * Save results for clients.
         */
        zsd_usage_cache_update(ctl);

        /*
         * Roll process accounting file.
         */
        (void) zsd_roll_exacct();
        return (0);
}

/*
 * Get the system rctl, which is the upper most limit
 */
static uint64_t
zsd_get_system_rctl(char *name)
{
        rctlblk_t *rblk, *rblk_last;

        rblk = (rctlblk_t *)alloca(rctlblk_size());
        rblk_last = (rctlblk_t *)alloca(rctlblk_size());

        if (getrctl(name, NULL, rblk_last, RCTL_FIRST) != 0)
                return (ZS_LIMIT_NONE);

        while (getrctl(name, rblk_last, rblk, RCTL_NEXT) == 0)
                (void) bcopy(rblk, rblk_last, rctlblk_size());

        return (rctlblk_get_value(rblk_last));
}

/*
 * Open any necessary subsystems for collecting utilization data,
 * allocate and initialize data structures, and get initial utilization.
 *
 * Errors:
 *      ENOMEM  out of memory
 *      EINVAL  other error
 */
static zsd_ctl_t *
zsd_open(zsd_ctl_t *ctl)
{
        zsd_system_t *system;

        char path[MAXPATHLEN];
        long pathmax;
        struct statvfs svfs;
        int ret;
        int i;
        size_t size;
        int err;

        if (ctl == NULL && (ctl = (zsd_ctl_t *)calloc(1,
            sizeof (zsd_ctl_t))) == NULL) {
                        zsd_warn(gettext("Out of Memory"));
                        errno = ENOMEM;
                        goto err;
        }
        ctl->zsctl_proc_fd = -1;

        /* open kstats */
        if (ctl->zsctl_kstat_ctl == NULL &&
            (ctl->zsctl_kstat_ctl = kstat_open()) == NULL) {
                err = errno;
                zsd_warn(gettext("Unable to open kstats"));
                errno = err;
                if (errno != ENOMEM)
                        errno = EAGAIN;
                goto err;
        }

        /*
         * These are set when the accounting file is opened by
         * zsd_update_procs()
         */
        ctl->zsctl_proc_fd = -1;
        ctl->zsctl_proc_fd_next = -1;
        ctl->zsctl_proc_open = 0;
        ctl->zsctl_proc_open_next = 0;

check_exacct:
        (void) zsd_enable_cpu_stats();

        /* Create structures to track usage */
        if (ctl->zsctl_system == NULL && (ctl->zsctl_system = (zsd_system_t *)
            calloc(1, sizeof (zsd_system_t))) == NULL) {
                ret = -1;
                zsd_warn(gettext("Out of Memory"));
                errno = ENOMEM;
                goto err;
        }
        system = ctl->zsctl_system;
        /* get the kernel bitness to know structure layout for getvmusage */
        ret = sysinfo(SI_ARCHITECTURE_64, path, sizeof (path));
        if (ret < 0)
                ctl->zsctl_kern_bits = 32;
        else
                ctl->zsctl_kern_bits = 64;
        ctl->zsctl_pagesize = sysconf(_SC_PAGESIZE);

        size = sysconf(_SC_CPUID_MAX);
        ctl->zsctl_maxcpuid = size;
        if (ctl->zsctl_cpu_array == NULL && (ctl->zsctl_cpu_array =
            (zsd_cpu_t *)calloc(size + 1, sizeof (zsd_cpu_t))) == NULL) {
                zsd_warn(gettext("Out of Memory"));
                errno = ENOMEM;
                goto err;
        }
        for (i = 0; i <= ctl->zsctl_maxcpuid; i++) {
                ctl->zsctl_cpu_array[i].zsc_id = i;
                ctl->zsctl_cpu_array[i].zsc_allocated = B_FALSE;
                ctl->zsctl_cpu_array[i].zsc_psetid = ZS_PSET_ERROR;
                ctl->zsctl_cpu_array[i].zsc_psetid_prev = ZS_PSET_ERROR;
        }
        if (statvfs("/proc", &svfs) != 0 ||
            strcmp("/proc", svfs.f_fstr) != 0) {
                zsd_warn(gettext("/proc not a procfs filesystem"));
                errno = EINVAL;
                goto err;
        }

        size = sysconf(_SC_MAXPID) + 1;
        ctl->zsctl_maxproc = size;
        if (ctl->zsctl_proc_array == NULL &&
            (ctl->zsctl_proc_array = (zsd_proc_t *)calloc(size,
            sizeof (zsd_proc_t))) == NULL) {
                zsd_warn(gettext("Out of Memory"));
                errno = ENOMEM;
                goto err;
        }
        for (i = 0; i <= ctl->zsctl_maxproc; i++) {
                list_link_init(&(ctl->zsctl_proc_array[i].zspr_next));
                ctl->zsctl_proc_array[i].zspr_psetid = ZS_PSET_ERROR;
                ctl->zsctl_proc_array[i].zspr_zoneid = -1;
                ctl->zsctl_proc_array[i].zspr_usage.tv_sec = 0;
                ctl->zsctl_proc_array[i].zspr_usage.tv_nsec = 0;
                ctl->zsctl_proc_array[i].zspr_ppid = -1;
        }

        list_create(&ctl->zsctl_zones, sizeof (zsd_zone_t),
            offsetof(zsd_zone_t, zsz_next));

        list_create(&ctl->zsctl_psets, sizeof (zsd_pset_t),
            offsetof(zsd_pset_t, zsp_next));

        list_create(&ctl->zsctl_cpus, sizeof (zsd_cpu_t),
            offsetof(zsd_cpu_t, zsc_next));

        pathmax = pathconf("/proc", _PC_NAME_MAX);
        if (pathmax < 0) {
                zsd_warn(gettext("Unable to determine max path of /proc"));
                errno = EINVAL;
                goto err;
        }
        size = sizeof (struct dirent) + pathmax + 1;

        ctl->zsctl_procfs_dent_size = size;
        if (ctl->zsctl_procfs_dent == NULL &&
            (ctl->zsctl_procfs_dent = (struct dirent *)calloc(1, size))
            == NULL) {
                zsd_warn(gettext("Out of Memory"));
                errno = ENOMEM;
                goto err;
        }

        if (ctl->zsctl_pool_conf == NULL &&
            (ctl->zsctl_pool_conf = pool_conf_alloc()) == NULL) {
                zsd_warn(gettext("Out of Memory"));
                errno = ENOMEM;
                goto err;
        }
        ctl->zsctl_pool_status = POOL_DISABLED;
        ctl->zsctl_pool_changed = 0;

        if (ctl->zsctl_pool_vals[0] == NULL &&
            (ctl->zsctl_pool_vals[0] = pool_value_alloc()) == NULL) {
                zsd_warn(gettext("Out of Memory"));
                errno = ENOMEM;
                goto err;
        }
        if (ctl->zsctl_pool_vals[1] == NULL &&
            (ctl->zsctl_pool_vals[1] = pool_value_alloc()) == NULL) {
                zsd_warn(gettext("Out of Memory"));
                errno = ENOMEM;
                goto err;
        }
        ctl->zsctl_pool_vals[2] = NULL;

        /*
         * get system limits
         */
        system->zss_maxpid = size = sysconf(_SC_MAXPID);
        system->zss_processes_max = zsd_get_system_rctl("zone.max-processes");
        system->zss_lwps_max = zsd_get_system_rctl("zone.max-lwps");
        system->zss_shm_max = zsd_get_system_rctl("zone.max-shm-memory");
        system->zss_shmids_max = zsd_get_system_rctl("zone.max-shm-ids");
        system->zss_semids_max = zsd_get_system_rctl("zone.max-sem-ids");
        system->zss_msgids_max = zsd_get_system_rctl("zone.max-msg-ids");
        system->zss_lofi_max = zsd_get_system_rctl("zone.max-lofi");

        g_gen_next = 1;

        if (zsd_read(ctl, B_TRUE, B_FALSE) != 0)
                zsd_warn(gettext("Reading zone statistics failed"));

        return (ctl);
err:
        if (ctl)
                zsd_close(ctl);

        return (NULL);
}

/* Copy utilization data to buffer, filtering data if non-global zone. */
static void
zsd_usage_filter(zoneid_t zid, zs_usage_cache_t *cache, zs_usage_t *usage,
    boolean_t is_gz)
{
        zs_usage_t *cusage;
        zs_system_t *sys, *csys;
        zs_zone_t *zone, *czone;
        zs_pset_t *pset, *cpset;
        zs_pset_zone_t *pz, *cpz, *foundpz;
        size_t size = 0, csize = 0;
        char *start, *cstart;
        int i, j;
        timestruc_t delta;

        /* Privileged users in the global zone get everything */
        if (is_gz) {
                cusage = cache->zsuc_usage;
                (void) bcopy(cusage, usage, cusage->zsu_size);
                return;
        }

        /* Zones just get their own usage */
        cusage = cache->zsuc_usage;

        start = (char *)usage;
        cstart = (char *)cusage;
        size += sizeof (zs_usage_t);
        csize += sizeof (zs_usage_t);

        usage->zsu_start = cusage->zsu_start;
        usage->zsu_hrstart = cusage->zsu_hrstart;
        usage->zsu_time = cusage->zsu_time;
        usage->zsu_hrtime = cusage->zsu_hrtime;
        usage->zsu_gen = cusage->zsu_gen;
        usage->zsu_nzones = 1;
        usage->zsu_npsets = 0;

        /* LINTED */
        sys = (zs_system_t *)(start + size);
        /* LINTED */
        csys = (zs_system_t *)(cstart + csize);
        size += sizeof (zs_system_t);
        csize += sizeof (zs_system_t);

        /* Save system limits but not usage */
        *sys = *csys;
        sys->zss_ncpus = 0;
        sys->zss_ncpus_online = 0;

        /* LINTED */
        zone = (zs_zone_t *)(start + size);
        /* LINTED */
        czone = (zs_zone_t *)(cstart + csize);
        /* Find the matching zone */
        for (i = 0; i < cusage->zsu_nzones; i++) {
                if (czone->zsz_id == zid) {
                        *zone = *czone;
                        size += sizeof (zs_zone_t);
                }
                csize += sizeof (zs_zone_t);
                /* LINTED */
                czone = (zs_zone_t *)(cstart + csize);
        }
        sys->zss_ram_kern += (sys->zss_ram_zones - zone->zsz_usage_ram);
        sys->zss_ram_zones = zone->zsz_usage_ram;

        sys->zss_vm_kern += (sys->zss_vm_zones - zone->zsz_usage_vm);
        sys->zss_vm_zones = zone->zsz_usage_vm;

        sys->zss_locked_kern += (sys->zss_locked_zones -
            zone->zsz_usage_locked);
        sys->zss_locked_zones = zone->zsz_usage_locked;

        TIMESTRUC_DELTA(delta, sys->zss_cpu_usage_zones, zone->zsz_cpu_usage);
        TIMESTRUC_ADD_TIMESTRUC(sys->zss_cpu_usage_kern, delta);
        sys->zss_cpu_usage_zones = zone->zsz_cpu_usage;

        /* LINTED */
        pset = (zs_pset_t *)(start + size);
        /* LINTED */
        cpset = (zs_pset_t *)(cstart + csize);
        for (i = 0; i < cusage->zsu_npsets; i++) {
                csize += sizeof (zs_pset_t);
                /* LINTED */
                cpz = (zs_pset_zone_t *)(csize + cstart);
                foundpz = NULL;
                for (j = 0; j < cpset->zsp_nusage; j++) {
                        if (cpz->zspz_zoneid == zid)
                                foundpz = cpz;

                        csize += sizeof (zs_pset_zone_t);
                        /* LINTED */
                        cpz = (zs_pset_zone_t *)(csize + cstart);
                }
                if (foundpz != NULL) {
                        size += sizeof (zs_pset_t);
                        /* LINTED */
                        pz = (zs_pset_zone_t *)(start + size);
                        size += sizeof (zs_pset_zone_t);

                        *pset = *cpset;
                        *pz = *foundpz;

                        TIMESTRUC_DELTA(delta, pset->zsp_usage_zones,
                            pz->zspz_cpu_usage);
                        TIMESTRUC_ADD_TIMESTRUC(pset->zsp_usage_kern, delta);
                        pset->zsp_usage_zones = pz->zspz_cpu_usage;
                        pset->zsp_nusage = 1;
                        usage->zsu_npsets++;
                        sys->zss_ncpus += pset->zsp_size;
                        sys->zss_ncpus_online += pset->zsp_online;
                }
                /* LINTED */
                cpset = (zs_pset_t *)(cstart + csize);
        }
        usage->zsu_size = size;
}

/*
 * Respond to new connections from libzonestat.so.  Also respond to zoneadmd,
 * which reports new zones.
 */
/* ARGSUSED */
static void
zsd_server(void *cookie, char *argp, size_t arg_size,
    door_desc_t *dp, uint_t n_desc)
{
        int *args, cmd;
        door_desc_t door;
        ucred_t *ucred;
        const priv_set_t *eset;

        if (argp == DOOR_UNREF_DATA) {
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }

        if (arg_size != sizeof (cmd) * 2) {
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }

        /* LINTED */
        args = (int *)argp;
        cmd = args[0];

        /* If connection, return door to stat server */
        if (cmd == ZSD_CMD_CONNECT) {

                /* Verify client compilation version */
                if (args[1] != ZS_VERSION) {
                        args[1] = ZSD_STATUS_VERSION_MISMATCH;
                        (void) door_return(argp, sizeof (cmd) * 2, NULL, 0);
                        thr_exit(NULL);
                }
                ucred = alloca(ucred_size());
                /* Verify client permission */
                if (door_ucred(&ucred) != 0) {
                        args[1] = ZSD_STATUS_INTERNAL_ERROR;
                        (void) door_return(argp, sizeof (cmd) * 2, NULL, 0);
                        thr_exit(NULL);
                }

                eset = ucred_getprivset(ucred, PRIV_EFFECTIVE);
                if (eset == NULL) {
                        args[1] = ZSD_STATUS_INTERNAL_ERROR;
                        (void) door_return(argp, sizeof (cmd) * 2, NULL, 0);
                        thr_exit(NULL);
                }
                if (!priv_ismember(eset, PRIV_PROC_INFO)) {
                        args[1] = ZSD_STATUS_PERMISSION;
                        (void) door_return(argp, sizeof (cmd) * 2, NULL, 0);
                        thr_exit(NULL);
                }

                /* Return stat server door */
                args[1] = ZSD_STATUS_OK;
                door.d_attributes = DOOR_DESCRIPTOR;
                door.d_data.d_desc.d_descriptor = g_stat_door;
                (void) door_return(argp, sizeof (cmd) * 2, &door, 1);
                thr_exit(NULL);
        }

        /* Respond to zoneadmd informing zonestatd of a new zone */
        if (cmd == ZSD_CMD_NEW_ZONE) {
                zsd_fattach_zone(args[1], g_server_door, B_FALSE);
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }

        args[1] = ZSD_STATUS_INTERNAL_ERROR;
        (void) door_return(argp, sizeof (cmd) * 2, NULL, 0);
        thr_exit(NULL);
}

/*
 * Respond to libzonestat.so clients with the current utlilzation data.
 */
/* ARGSUSED */
static void
zsd_stat_server(void *cookie, char *argp, size_t arg_size,
    door_desc_t *dp, uint_t n_desc)
{
        uint64_t *args, cmd;
        zs_usage_cache_t *cache;
        int ret;
        char *rvalp;
        size_t rvals;
        zs_usage_t *usage;
        ucred_t *ucred;
        zoneid_t zoneid;
        const priv_set_t *eset;
        boolean_t is_gz = B_FALSE;

        /* Tell stat thread there are no more clients */
        if (argp == DOOR_UNREF_DATA) {
                (void) mutex_lock(&g_usage_cache_lock);
                g_hasclient = B_FALSE;
                (void) cond_signal(&g_usage_cache_kick);
                (void) mutex_unlock(&g_usage_cache_lock);
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        if (arg_size != sizeof (cmd) * 2) {
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        /* LINTED */
        args = (uint64_t *)argp;
        cmd = args[0];
        if (cmd != ZSD_CMD_READ) {
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        ucred = alloca(ucred_size());
        if (door_ucred(&ucred) != 0) {
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        zoneid = ucred_getzoneid(ucred);

        if (zoneid == GLOBAL_ZONEID)
                is_gz = B_TRUE;

        eset = ucred_getprivset(ucred, PRIV_EFFECTIVE);
        if (eset == NULL) {
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        if (!priv_ismember(eset, PRIV_PROC_INFO)) {
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        (void) mutex_lock(&g_usage_cache_lock);
        g_hasclient = B_TRUE;

        /*
         * Force a new cpu calculation for client.  This will force a
         * new memory calculation if the memory data is older than the
         * sample period.
         */
        g_usage_cache_kickers++;
        (void) cond_signal(&g_usage_cache_kick);
        ret = cond_wait(&g_usage_cache_wait, &g_usage_cache_lock);
        g_usage_cache_kickers--;
        if (ret != 0 && errno == EINTR) {
                (void) mutex_unlock(&g_usage_cache_lock);
                zsd_warn(gettext(
                    "Interrupted before writing usage size to client\n"));
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        cache = zsd_usage_cache_hold_locked();
        if (cache == NULL) {
                zsd_warn(gettext("Usage cache empty.\n"));
                (void) door_return(NULL, 0, NULL, 0);
                thr_exit(NULL);
        }
        (void) mutex_unlock(&g_usage_cache_lock);

        /* Copy current usage data to stack to send to client */
        usage = (zs_usage_t *)alloca(cache->zsuc_size);

        /* Filter out results if caller is non-global zone */
        zsd_usage_filter(zoneid, cache, usage, is_gz);

        rvalp = (void *)usage;
        rvals = usage->zsu_size;
        zsd_usage_cache_rele(cache);

        (void) door_return(rvalp, rvals, 0, NULL);
        thr_exit(NULL);
}

static volatile boolean_t g_quit;

/* ARGSUSED */
static void
zonestat_quithandler(int sig)
{
        g_quit = B_TRUE;
}

/*
 * The stat thread generates new utilization data when clients request
 * it.  It also manages opening and closing the subsystems used to gather
 * data depending on if clients exist.
 */
/* ARGSUSED */
void *
stat_thread(void *arg)
{
        time_t start;
        time_t now;
        time_t next_memory;
        boolean_t do_memory;
        boolean_t do_read;
        boolean_t do_close;

        start = time(NULL);
        if (start < 0) {
                if (g_quit == B_TRUE)
                        goto quit;
                zsd_warn(gettext("Unable to fetch current time"));
                g_quit = B_TRUE;
                goto quit;
        }

        next_memory = start;
        while (g_quit == B_FALSE) {
                for (;;) {
                        /*
                         * These are used to decide if the most recent memory
                         * calculation was within a sample interval,
                         * and weather or not the usage collection needs to
                         * be opened or closed.
                         */
                        do_memory = B_FALSE;
                        do_read = B_FALSE;
                        do_close = B_FALSE;

                        /*
                         * If all clients have gone, close usage collecting
                         */
                        (void) mutex_lock(&g_usage_cache_lock);
                        if (!g_hasclient && g_open == B_TRUE) {
                                do_close = B_TRUE;
                                (void) mutex_unlock(&g_usage_cache_lock);
                                break;
                        }
                        if (g_quit == B_TRUE) {
                                (void) mutex_unlock(
                                    &g_usage_cache_lock);
                                break;
                        }
                        /*
                         * Wait for a usage data request
                         */
                        if (g_usage_cache_kickers == 0) {
                                (void) cond_wait(&g_usage_cache_kick,
                                    &g_usage_cache_lock);
                        }
                        now = time(NULL);
                        if (now < 0) {
                                if (g_quit == B_TRUE) {
                                        (void) mutex_unlock(
                                            &g_usage_cache_lock);
                                        goto quit;
                                }
                                g_quit = B_TRUE;
                                (void) mutex_unlock(&g_usage_cache_lock);
                                zsd_warn(gettext(
                                    "Unable to fetch current time"));
                                goto quit;
                        }
                        if (g_hasclient) {
                                do_read = B_TRUE;
                                if (now >= next_memory) {
                                        do_memory = B_TRUE;
                                        next_memory = now + g_interval;
                                }
                        } else {
                                do_close = B_TRUE;
                        }
                        (void) mutex_unlock(&g_usage_cache_lock);
                        if (do_read || do_close)
                                break;
                }
                g_now = now;
                g_hrnow = gethrtime();
                if (g_hasclient && g_open == B_FALSE) {
                        g_start = g_now;
                        g_hrstart = g_hrnow;
                        g_ctl = zsd_open(g_ctl);
                        if (g_ctl == NULL)
                                zsd_warn(gettext(
                                    "Unable to open zone statistics"));
                        else
                                g_open = B_TRUE;
                }
                if (do_read && g_ctl) {
                        if (zsd_read(g_ctl, B_FALSE, do_memory) != 0) {
                                zsd_warn(gettext(
                                    "Unable to read zone statistics"));
                                g_quit = B_TRUE;
                                return (NULL);
                        }
                }
                (void) mutex_lock(&g_usage_cache_lock);
                if (!g_hasclient && g_open == B_TRUE && g_ctl) {
                        (void) mutex_unlock(&g_usage_cache_lock);
                        zsd_close(g_ctl);
                        g_open = B_FALSE;
                } else {
                        (void) mutex_unlock(&g_usage_cache_lock);
                }
        }
quit:
        if (g_open)
                zsd_close(g_ctl);

        (void) thr_kill(g_main, SIGINT);
        thr_exit(NULL);
        return (NULL);
}

void
zsd_set_fx()
{
        pcinfo_t pcinfo;
        pcparms_t pcparms;

        (void) strlcpy(pcinfo.pc_clname, "FX", sizeof (pcinfo.pc_clname));
        if (priocntl(0, 0, PC_GETCID, (caddr_t)&pcinfo) == -1) {
                zsd_warn(gettext("cannot get FX class parameters"));
                return;
        }
        pcparms.pc_cid = pcinfo.pc_cid;
        ((fxparms_t *)pcparms.pc_clparms)->fx_upri = 60;
        ((fxparms_t *)pcparms.pc_clparms)->fx_uprilim = 60;
        ((fxparms_t *)pcparms.pc_clparms)->fx_tqsecs = 0;
        ((fxparms_t *)pcparms.pc_clparms)->fx_tqnsecs = FX_NOCHANGE;
        if (priocntl(P_PID, getpid(), PC_SETPARMS, (caddr_t)&pcparms) == -1)
                zsd_warn(gettext("cannot enter the FX class"));
}

static int pipe_fd;

static void
daemonize_ready(char status)
{
        /*
         * wake the parent with a clue
         */
        (void) write(pipe_fd, &status, 1);
        (void) close(pipe_fd);
}

static int
daemonize_start(void)
{
        char data;
        int status;

        int filedes[2];
        pid_t pid;

        (void) close(0);
        (void) dup2(2, 1);

        if (pipe(filedes) < 0)
                return (-1);

        (void) fflush(NULL);

        if ((pid = fork1()) < 0)
                return (-1);

        if (pid != 0) {
                /*
                 * parent
                 */
                struct sigaction act;

                act.sa_sigaction = SIG_DFL;
                (void) sigemptyset(&act.sa_mask);
                act.sa_flags = 0;

                (void) sigaction(SIGPIPE, &act, NULL);  /* ignore SIGPIPE */

                (void) close(filedes[1]);
                if (read(filedes[0], &data, 1) == 1) {
                        /* forward ready code via exit status */
                        exit(data);
                }
                status = -1;
                (void) wait4(pid, &status, 0, NULL);
                /* daemon process exited before becoming ready */
                if (WIFEXITED(status)) {
                        /* assume daemon process printed useful message */
                        exit(WEXITSTATUS(status));
                } else {
                        zsd_warn(gettext("daemon process killed or died"));
                        exit(1);
                }
        }

        /*
         * child
         */
        pipe_fd = filedes[1];
        (void) close(filedes[0]);

        /*
         * generic Unix setup
         */
        (void) setsid();
        (void) umask(0000);

        return (0);
}

static void
fattach_all_zones(boolean_t detach_only)
{
        zoneid_t *zids;
        uint_t nzids, nzids_last;
        int i;

again:
        (void) zone_list(NULL, &nzids);
        nzids_last = nzids;
        zids = (zoneid_t *)malloc(sizeof (zoneid_t) * nzids_last);
        if (zids == NULL)
                zsd_error(gettext("Out of memory"));

        (void) zone_list(zids, &nzids);
        if (nzids > nzids_last) {
                free(zids);
                goto again;
        }
        for (i = 0; i < nzids; i++)
                zsd_fattach_zone(zids[i], g_server_door, detach_only);

        free(zids);
}

int
main(int argc, char *argv[])
{

        int arg;
        thread_t tid;
        scf_simple_prop_t *prop;
        uint64_t *intervalp;
        boolean_t opt_cleanup = B_FALSE;

        g_main = thr_self();
        g_quit = B_FALSE;
        (void) signal(SIGINT, zonestat_quithandler);
        (void) signal(SIGTERM, zonestat_quithandler);
        (void) signal(SIGHUP, zonestat_quithandler);
/*      (void) sigignore(SIGCHLD); */
        (void) sigignore(SIGPIPE);

        if (getzoneid() != GLOBAL_ZONEID)
                zsd_error(gettext("Must be run from global zone only"));

        while ((arg = getopt(argc, argv, "c"))
            != EOF) {
                switch (arg) {
                case 'c':
                        opt_cleanup = B_TRUE;
                        break;
                default:
                        zsd_error(gettext("Invalid option"));
                }
        }

        if (opt_cleanup) {
                if (zsd_disable_cpu_stats() != 0)
                        exit(1);
                else
                        exit(0);
        }

        /* Get the configured sample interval */
        prop = scf_simple_prop_get(NULL, "svc:/system/zones-monitoring:default",
            "config", "sample_interval");
        if (prop == NULL)
                zsd_error(gettext("Unable to fetch SMF property "
                    "\"config/sample_interval\""));

        if (scf_simple_prop_type(prop) != SCF_TYPE_COUNT)
                zsd_error(gettext("Malformed SMF property "
                    "\"config/sample_interval\".  Must be of type \"count\""));

        intervalp = scf_simple_prop_next_count(prop);
        g_interval = *intervalp;
        if (g_interval == 0)
                zsd_error(gettext("Malformed SMF property "
                    "\"config/sample_interval\".  Must be greater than zero"));

        scf_simple_prop_free(prop);

        if (daemonize_start() < 0)
                zsd_error(gettext("Unable to start daemon\n"));

        /* Run at high priority */
        zsd_set_fx();

        (void) mutex_init(&g_usage_cache_lock, USYNC_THREAD, NULL);
        (void) cond_init(&g_usage_cache_kick, USYNC_THREAD, NULL);
        (void) cond_init(&g_usage_cache_wait, USYNC_THREAD, NULL);

        g_server_door = door_create(zsd_server, NULL,
            DOOR_REFUSE_DESC | DOOR_NO_CANCEL);
        if (g_server_door < 0)
                zsd_error(gettext("Unable to create server door\n"));


        g_stat_door = door_create(zsd_stat_server, NULL, DOOR_UNREF_MULTI |
            DOOR_REFUSE_DESC | DOOR_NO_CANCEL);
        if (g_stat_door < 0)
                zsd_error(gettext("Unable to create statistics door\n"));

        fattach_all_zones(B_FALSE);

        if (thr_create(NULL, 0, stat_thread, NULL, 0, &tid) != 0)
                zsd_error(gettext("Unable to create statistics thread\n"));

        daemonize_ready(0);

        /* Wait for signal to quit */
        while (g_quit == B_FALSE)
                (void) pause();

        /* detach doors */
        fattach_all_zones(B_TRUE);

        (void) door_revoke(g_server_door);
        (void) door_revoke(g_stat_door);

        /* kick stat thread and wait for it to close the statistics */
        (void) mutex_lock(&g_usage_cache_lock);
        g_quit = B_TRUE;
        (void) cond_signal(&g_usage_cache_kick);
        (void) mutex_unlock(&g_usage_cache_lock);
end:
        (void) thr_join(tid, NULL, NULL);
        return (0);
}