4 * Manages VM statistics
5 * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
8 * Copyright (C) 2006 Silicon Graphics, Inc.,
9 * Christoph Lameter <christoph@lameter.com>
13 #include <linux/err.h>
14 #include <linux/module.h>
15 #include <linux/cpu.h>
16 #include <linux/sched.h>
18 #ifdef CONFIG_VM_EVENT_COUNTERS
19 DEFINE_PER_CPU(struct vm_event_state
, vm_event_states
) = {{0}};
20 EXPORT_PER_CPU_SYMBOL(vm_event_states
);
22 static void sum_vm_events(unsigned long *ret
, cpumask_t
*cpumask
)
27 memset(ret
, 0, NR_VM_EVENT_ITEMS
* sizeof(unsigned long));
29 cpu
= first_cpu(*cpumask
);
30 while (cpu
< NR_CPUS
) {
31 struct vm_event_state
*this = &per_cpu(vm_event_states
, cpu
);
33 cpu
= next_cpu(cpu
, *cpumask
);
36 prefetch(&per_cpu(vm_event_states
, cpu
));
39 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++)
40 ret
[i
] += this->event
[i
];
45 * Accumulate the vm event counters across all CPUs.
46 * The result is unavoidably approximate - it can change
47 * during and after execution of this function.
49 void all_vm_events(unsigned long *ret
)
51 sum_vm_events(ret
, &cpu_online_map
);
53 EXPORT_SYMBOL_GPL(all_vm_events
);
57 * Fold the foreign cpu events into our own.
59 * This is adding to the events on one processor
60 * but keeps the global counts constant.
62 void vm_events_fold_cpu(int cpu
)
64 struct vm_event_state
*fold_state
= &per_cpu(vm_event_states
, cpu
);
67 for (i
= 0; i
< NR_VM_EVENT_ITEMS
; i
++) {
68 count_vm_events(i
, fold_state
->event
[i
]);
69 fold_state
->event
[i
] = 0;
72 #endif /* CONFIG_HOTPLUG */
74 #endif /* CONFIG_VM_EVENT_COUNTERS */
77 * Manage combined zone based / global counters
79 * vm_stat contains the global counters
81 atomic_long_t vm_stat
[NR_VM_ZONE_STAT_ITEMS
];
82 EXPORT_SYMBOL(vm_stat
);
86 static int calculate_threshold(struct zone
*zone
)
89 int mem
; /* memory in 128 MB units */
92 * The threshold scales with the number of processors and the amount
93 * of memory per zone. More memory means that we can defer updates for
94 * longer, more processors could lead to more contention.
95 * fls() is used to have a cheap way of logarithmic scaling.
97 * Some sample thresholds:
99 * Threshold Processors (fls) Zonesize fls(mem+1)
100 * ------------------------------------------------------------------
117 * 125 1024 10 8-16 GB 8
118 * 125 1024 10 16-32 GB 9
121 mem
= zone
->present_pages
>> (27 - PAGE_SHIFT
);
123 threshold
= 2 * fls(num_online_cpus()) * (1 + fls(mem
));
126 * Maximum threshold is 125
128 threshold
= min(125, threshold
);
134 * Refresh the thresholds for each zone.
136 static void refresh_zone_stat_thresholds(void)
142 for_each_zone(zone
) {
144 if (!zone
->present_pages
)
147 threshold
= calculate_threshold(zone
);
149 for_each_online_cpu(cpu
)
150 zone_pcp(zone
, cpu
)->stat_threshold
= threshold
;
155 * For use when we know that interrupts are disabled.
157 void __mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
160 struct per_cpu_pageset
*pcp
= zone_pcp(zone
, smp_processor_id());
161 s8
*p
= pcp
->vm_stat_diff
+ item
;
166 if (unlikely(x
> pcp
->stat_threshold
|| x
< -pcp
->stat_threshold
)) {
167 zone_page_state_add(x
, zone
, item
);
172 EXPORT_SYMBOL(__mod_zone_page_state
);
175 * For an unknown interrupt state
177 void mod_zone_page_state(struct zone
*zone
, enum zone_stat_item item
,
182 local_irq_save(flags
);
183 __mod_zone_page_state(zone
, item
, delta
);
184 local_irq_restore(flags
);
186 EXPORT_SYMBOL(mod_zone_page_state
);
189 * Optimized increment and decrement functions.
191 * These are only for a single page and therefore can take a struct page *
192 * argument instead of struct zone *. This allows the inclusion of the code
193 * generated for page_zone(page) into the optimized functions.
195 * No overflow check is necessary and therefore the differential can be
196 * incremented or decremented in place which may allow the compilers to
197 * generate better code.
198 * The increment or decrement is known and therefore one boundary check can
201 * NOTE: These functions are very performance sensitive. Change only
204 * Some processors have inc/dec instructions that are atomic vs an interrupt.
205 * However, the code must first determine the differential location in a zone
206 * based on the processor number and then inc/dec the counter. There is no
207 * guarantee without disabling preemption that the processor will not change
208 * in between and therefore the atomicity vs. interrupt cannot be exploited
209 * in a useful way here.
211 void __inc_zone_state(struct zone
*zone
, enum zone_stat_item item
)
213 struct per_cpu_pageset
*pcp
= zone_pcp(zone
, smp_processor_id());
214 s8
*p
= pcp
->vm_stat_diff
+ item
;
218 if (unlikely(*p
> pcp
->stat_threshold
)) {
219 int overstep
= pcp
->stat_threshold
/ 2;
221 zone_page_state_add(*p
+ overstep
, zone
, item
);
226 void __inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
228 __inc_zone_state(page_zone(page
), item
);
230 EXPORT_SYMBOL(__inc_zone_page_state
);
232 void __dec_zone_state(struct zone
*zone
, enum zone_stat_item item
)
234 struct per_cpu_pageset
*pcp
= zone_pcp(zone
, smp_processor_id());
235 s8
*p
= pcp
->vm_stat_diff
+ item
;
239 if (unlikely(*p
< - pcp
->stat_threshold
)) {
240 int overstep
= pcp
->stat_threshold
/ 2;
242 zone_page_state_add(*p
- overstep
, zone
, item
);
247 void __dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
249 __dec_zone_state(page_zone(page
), item
);
251 EXPORT_SYMBOL(__dec_zone_page_state
);
253 void inc_zone_state(struct zone
*zone
, enum zone_stat_item item
)
257 local_irq_save(flags
);
258 __inc_zone_state(zone
, item
);
259 local_irq_restore(flags
);
262 void inc_zone_page_state(struct page
*page
, enum zone_stat_item item
)
267 zone
= page_zone(page
);
268 local_irq_save(flags
);
269 __inc_zone_state(zone
, item
);
270 local_irq_restore(flags
);
272 EXPORT_SYMBOL(inc_zone_page_state
);
274 void dec_zone_page_state(struct page
*page
, enum zone_stat_item item
)
278 local_irq_save(flags
);
279 __dec_zone_page_state(page
, item
);
280 local_irq_restore(flags
);
282 EXPORT_SYMBOL(dec_zone_page_state
);
285 * Update the zone counters for one cpu.
287 * The cpu specified must be either the current cpu or a processor that
288 * is not online. If it is the current cpu then the execution thread must
289 * be pinned to the current cpu.
291 * Note that refresh_cpu_vm_stats strives to only access
292 * node local memory. The per cpu pagesets on remote zones are placed
293 * in the memory local to the processor using that pageset. So the
294 * loop over all zones will access a series of cachelines local to
297 * The call to zone_page_state_add updates the cachelines with the
298 * statistics in the remote zone struct as well as the global cachelines
299 * with the global counters. These could cause remote node cache line
300 * bouncing and will have to be only done when necessary.
302 void refresh_cpu_vm_stats(int cpu
)
306 int global_diff
[NR_VM_ZONE_STAT_ITEMS
] = { 0, };
308 for_each_zone(zone
) {
309 struct per_cpu_pageset
*p
;
311 if (!populated_zone(zone
))
314 p
= zone_pcp(zone
, cpu
);
316 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
317 if (p
->vm_stat_diff
[i
]) {
321 local_irq_save(flags
);
322 v
= p
->vm_stat_diff
[i
];
323 p
->vm_stat_diff
[i
] = 0;
324 local_irq_restore(flags
);
325 atomic_long_add(v
, &zone
->vm_stat
[i
]);
328 /* 3 seconds idle till flush */
334 * Deal with draining the remote pageset of this
337 * Check if there are pages remaining in this pageset
338 * if not then there is nothing to expire.
340 if (!p
->expire
|| (!p
->pcp
[0].count
&& !p
->pcp
[1].count
))
344 * We never drain zones local to this processor.
346 if (zone_to_nid(zone
) == numa_node_id()) {
356 drain_zone_pages(zone
, p
->pcp
+ 0);
359 drain_zone_pages(zone
, p
->pcp
+ 1);
363 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
365 atomic_long_add(global_diff
[i
], &vm_stat
[i
]);
372 * zonelist = the list of zones passed to the allocator
373 * z = the zone from which the allocation occurred.
375 * Must be called with interrupts disabled.
377 void zone_statistics(struct zonelist
*zonelist
, struct zone
*z
)
379 if (z
->zone_pgdat
== zonelist
->zones
[0]->zone_pgdat
) {
380 __inc_zone_state(z
, NUMA_HIT
);
382 __inc_zone_state(z
, NUMA_MISS
);
383 __inc_zone_state(zonelist
->zones
[0], NUMA_FOREIGN
);
385 if (z
->node
== numa_node_id())
386 __inc_zone_state(z
, NUMA_LOCAL
);
388 __inc_zone_state(z
, NUMA_OTHER
);
392 #ifdef CONFIG_PROC_FS
394 #include <linux/seq_file.h>
396 static char * const migratetype_names
[MIGRATE_TYPES
] = {
403 static void *frag_start(struct seq_file
*m
, loff_t
*pos
)
407 for (pgdat
= first_online_pgdat();
409 pgdat
= next_online_pgdat(pgdat
))
415 static void *frag_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
417 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
420 return next_online_pgdat(pgdat
);
423 static void frag_stop(struct seq_file
*m
, void *arg
)
427 /* Walk all the zones in a node and print using a callback */
428 static void walk_zones_in_node(struct seq_file
*m
, pg_data_t
*pgdat
,
429 void (*print
)(struct seq_file
*m
, pg_data_t
*, struct zone
*))
432 struct zone
*node_zones
= pgdat
->node_zones
;
435 for (zone
= node_zones
; zone
- node_zones
< MAX_NR_ZONES
; ++zone
) {
436 if (!populated_zone(zone
))
439 spin_lock_irqsave(&zone
->lock
, flags
);
440 print(m
, pgdat
, zone
);
441 spin_unlock_irqrestore(&zone
->lock
, flags
);
445 static void frag_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
450 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
451 for (order
= 0; order
< MAX_ORDER
; ++order
)
452 seq_printf(m
, "%6lu ", zone
->free_area
[order
].nr_free
);
457 * This walks the free areas for each zone.
459 static int frag_show(struct seq_file
*m
, void *arg
)
461 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
462 walk_zones_in_node(m
, pgdat
, frag_show_print
);
466 static void pagetypeinfo_showfree_print(struct seq_file
*m
,
467 pg_data_t
*pgdat
, struct zone
*zone
)
471 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++) {
472 seq_printf(m
, "Node %4d, zone %8s, type %12s ",
475 migratetype_names
[mtype
]);
476 for (order
= 0; order
< MAX_ORDER
; ++order
) {
477 unsigned long freecount
= 0;
478 struct free_area
*area
;
479 struct list_head
*curr
;
481 area
= &(zone
->free_area
[order
]);
483 list_for_each(curr
, &area
->free_list
[mtype
])
485 seq_printf(m
, "%6lu ", freecount
);
491 /* Print out the free pages at each order for each migatetype */
492 static int pagetypeinfo_showfree(struct seq_file
*m
, void *arg
)
495 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
498 seq_printf(m
, "%-43s ", "Free pages count per migrate type at order");
499 for (order
= 0; order
< MAX_ORDER
; ++order
)
500 seq_printf(m
, "%6d ", order
);
503 walk_zones_in_node(m
, pgdat
, pagetypeinfo_showfree_print
);
508 static void pagetypeinfo_showblockcount_print(struct seq_file
*m
,
509 pg_data_t
*pgdat
, struct zone
*zone
)
513 unsigned long start_pfn
= zone
->zone_start_pfn
;
514 unsigned long end_pfn
= start_pfn
+ zone
->spanned_pages
;
515 unsigned long count
[MIGRATE_TYPES
] = { 0, };
517 for (pfn
= start_pfn
; pfn
< end_pfn
; pfn
+= pageblock_nr_pages
) {
523 page
= pfn_to_page(pfn
);
524 mtype
= get_pageblock_migratetype(page
);
530 seq_printf(m
, "Node %d, zone %8s ", pgdat
->node_id
, zone
->name
);
531 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
532 seq_printf(m
, "%12lu ", count
[mtype
]);
536 /* Print out the free pages at each order for each migratetype */
537 static int pagetypeinfo_showblockcount(struct seq_file
*m
, void *arg
)
540 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
542 seq_printf(m
, "\n%-23s", "Number of blocks type ");
543 for (mtype
= 0; mtype
< MIGRATE_TYPES
; mtype
++)
544 seq_printf(m
, "%12s ", migratetype_names
[mtype
]);
546 walk_zones_in_node(m
, pgdat
, pagetypeinfo_showblockcount_print
);
552 * This prints out statistics in relation to grouping pages by mobility.
553 * It is expensive to collect so do not constantly read the file.
555 static int pagetypeinfo_show(struct seq_file
*m
, void *arg
)
557 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
559 seq_printf(m
, "Page block order: %d\n", pageblock_order
);
560 seq_printf(m
, "Pages per block: %lu\n", pageblock_nr_pages
);
562 pagetypeinfo_showfree(m
, pgdat
);
563 pagetypeinfo_showblockcount(m
, pgdat
);
568 const struct seq_operations fragmentation_op
= {
575 const struct seq_operations pagetypeinfo_op
= {
579 .show
= pagetypeinfo_show
,
582 #ifdef CONFIG_ZONE_DMA
583 #define TEXT_FOR_DMA(xx) xx "_dma",
585 #define TEXT_FOR_DMA(xx)
588 #ifdef CONFIG_ZONE_DMA32
589 #define TEXT_FOR_DMA32(xx) xx "_dma32",
591 #define TEXT_FOR_DMA32(xx)
594 #ifdef CONFIG_HIGHMEM
595 #define TEXT_FOR_HIGHMEM(xx) xx "_high",
597 #define TEXT_FOR_HIGHMEM(xx)
600 #define TEXTS_FOR_ZONES(xx) TEXT_FOR_DMA(xx) TEXT_FOR_DMA32(xx) xx "_normal", \
601 TEXT_FOR_HIGHMEM(xx) xx "_movable",
603 static const char * const vmstat_text
[] = {
604 /* Zoned VM counters */
613 "nr_slab_reclaimable",
614 "nr_slab_unreclaimable",
615 "nr_page_table_pages",
629 #ifdef CONFIG_VM_EVENT_COUNTERS
635 TEXTS_FOR_ZONES("pgalloc")
644 TEXTS_FOR_ZONES("pgrefill")
645 TEXTS_FOR_ZONES("pgsteal")
646 TEXTS_FOR_ZONES("pgscan_kswapd")
647 TEXTS_FOR_ZONES("pgscan_direct")
660 static void zoneinfo_show_print(struct seq_file
*m
, pg_data_t
*pgdat
,
664 seq_printf(m
, "Node %d, zone %8s", pgdat
->node_id
, zone
->name
);
670 "\n scanned %lu (a: %lu i: %lu)"
673 zone_page_state(zone
, NR_FREE_PAGES
),
678 zone
->nr_scan_active
, zone
->nr_scan_inactive
,
680 zone
->present_pages
);
682 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
683 seq_printf(m
, "\n %-12s %lu", vmstat_text
[i
],
684 zone_page_state(zone
, i
));
687 "\n protection: (%lu",
688 zone
->lowmem_reserve
[0]);
689 for (i
= 1; i
< ARRAY_SIZE(zone
->lowmem_reserve
); i
++)
690 seq_printf(m
, ", %lu", zone
->lowmem_reserve
[i
]);
694 for_each_online_cpu(i
) {
695 struct per_cpu_pageset
*pageset
;
698 pageset
= zone_pcp(zone
, i
);
699 for (j
= 0; j
< ARRAY_SIZE(pageset
->pcp
); j
++) {
706 pageset
->pcp
[j
].count
,
707 pageset
->pcp
[j
].high
,
708 pageset
->pcp
[j
].batch
);
711 seq_printf(m
, "\n vm stats threshold: %d",
712 pageset
->stat_threshold
);
716 "\n all_unreclaimable: %u"
717 "\n prev_priority: %i"
719 zone_is_all_unreclaimable(zone
),
721 zone
->zone_start_pfn
);
726 * Output information about zones in @pgdat.
728 static int zoneinfo_show(struct seq_file
*m
, void *arg
)
730 pg_data_t
*pgdat
= (pg_data_t
*)arg
;
731 walk_zones_in_node(m
, pgdat
, zoneinfo_show_print
);
735 const struct seq_operations zoneinfo_op
= {
736 .start
= frag_start
, /* iterate over all zones. The same as in
740 .show
= zoneinfo_show
,
743 static void *vmstat_start(struct seq_file
*m
, loff_t
*pos
)
746 #ifdef CONFIG_VM_EVENT_COUNTERS
751 if (*pos
>= ARRAY_SIZE(vmstat_text
))
754 #ifdef CONFIG_VM_EVENT_COUNTERS
755 v
= kmalloc(NR_VM_ZONE_STAT_ITEMS
* sizeof(unsigned long)
756 + sizeof(struct vm_event_state
), GFP_KERNEL
);
758 v
= kmalloc(NR_VM_ZONE_STAT_ITEMS
* sizeof(unsigned long),
763 return ERR_PTR(-ENOMEM
);
764 for (i
= 0; i
< NR_VM_ZONE_STAT_ITEMS
; i
++)
765 v
[i
] = global_page_state(i
);
766 #ifdef CONFIG_VM_EVENT_COUNTERS
767 e
= v
+ NR_VM_ZONE_STAT_ITEMS
;
769 e
[PGPGIN
] /= 2; /* sectors -> kbytes */
775 static void *vmstat_next(struct seq_file
*m
, void *arg
, loff_t
*pos
)
778 if (*pos
>= ARRAY_SIZE(vmstat_text
))
780 return (unsigned long *)m
->private + *pos
;
783 static int vmstat_show(struct seq_file
*m
, void *arg
)
785 unsigned long *l
= arg
;
786 unsigned long off
= l
- (unsigned long *)m
->private;
788 seq_printf(m
, "%s %lu\n", vmstat_text
[off
], *l
);
792 static void vmstat_stop(struct seq_file
*m
, void *arg
)
798 const struct seq_operations vmstat_op
= {
799 .start
= vmstat_start
,
805 #endif /* CONFIG_PROC_FS */
808 static DEFINE_PER_CPU(struct delayed_work
, vmstat_work
);
809 int sysctl_stat_interval __read_mostly
= HZ
;
811 static void vmstat_update(struct work_struct
*w
)
813 refresh_cpu_vm_stats(smp_processor_id());
814 schedule_delayed_work(&__get_cpu_var(vmstat_work
),
815 sysctl_stat_interval
);
818 static void __cpuinit
start_cpu_timer(int cpu
)
820 struct delayed_work
*vmstat_work
= &per_cpu(vmstat_work
, cpu
);
822 INIT_DELAYED_WORK_DEFERRABLE(vmstat_work
, vmstat_update
);
823 schedule_delayed_work_on(cpu
, vmstat_work
, HZ
+ cpu
);
827 * Use the cpu notifier to insure that the thresholds are recalculated
830 static int __cpuinit
vmstat_cpuup_callback(struct notifier_block
*nfb
,
831 unsigned long action
,
834 long cpu
= (long)hcpu
;
838 case CPU_ONLINE_FROZEN
:
839 start_cpu_timer(cpu
);
841 case CPU_DOWN_PREPARE
:
842 case CPU_DOWN_PREPARE_FROZEN
:
843 cancel_rearming_delayed_work(&per_cpu(vmstat_work
, cpu
));
844 per_cpu(vmstat_work
, cpu
).work
.func
= NULL
;
846 case CPU_DOWN_FAILED
:
847 case CPU_DOWN_FAILED_FROZEN
:
848 start_cpu_timer(cpu
);
851 case CPU_DEAD_FROZEN
:
852 refresh_zone_stat_thresholds();
860 static struct notifier_block __cpuinitdata vmstat_notifier
=
861 { &vmstat_cpuup_callback
, NULL
, 0 };
863 static int __init
setup_vmstat(void)
867 refresh_zone_stat_thresholds();
868 register_cpu_notifier(&vmstat_notifier
);
870 for_each_online_cpu(cpu
)
871 start_cpu_timer(cpu
);
874 module_init(setup_vmstat
)