2 * Copyright (c) 2012, Microsoft Corporation.
5 * K. Y. Srinivasan <kys@microsoft.com>
7 * This program is free software; you can redistribute it and/or modify it
8 * under the terms of the GNU General Public License version 2 as published
9 * by the Free Software Foundation.
11 * This program is distributed in the hope that it will be useful, but
12 * WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
14 * NON INFRINGEMENT. See the GNU General Public License for more
19 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
21 #include <linux/kernel.h>
22 #include <linux/mman.h>
23 #include <linux/delay.h>
24 #include <linux/init.h>
25 #include <linux/module.h>
26 #include <linux/slab.h>
27 #include <linux/kthread.h>
28 #include <linux/completion.h>
29 #include <linux/memory_hotplug.h>
30 #include <linux/memory.h>
31 #include <linux/notifier.h>
32 #include <linux/percpu_counter.h>
34 #include <linux/hyperv.h>
37 * We begin with definitions supporting the Dynamic Memory protocol
40 * Begin protocol definitions.
46 * Protocol versions. The low word is the minor version, the high word the major
51 * Changed to 0.1 on 2009/03/25
52 * Changes to 0.2 on 2009/05/14
53 * Changes to 0.3 on 2009/12/03
54 * Changed to 1.0 on 2011/04/05
57 #define DYNMEM_MAKE_VERSION(Major, Minor) ((__u32)(((Major) << 16) | (Minor)))
58 #define DYNMEM_MAJOR_VERSION(Version) ((__u32)(Version) >> 16)
59 #define DYNMEM_MINOR_VERSION(Version) ((__u32)(Version) & 0xff)
62 DYNMEM_PROTOCOL_VERSION_1
= DYNMEM_MAKE_VERSION(0, 3),
63 DYNMEM_PROTOCOL_VERSION_2
= DYNMEM_MAKE_VERSION(1, 0),
65 DYNMEM_PROTOCOL_VERSION_WIN7
= DYNMEM_PROTOCOL_VERSION_1
,
66 DYNMEM_PROTOCOL_VERSION_WIN8
= DYNMEM_PROTOCOL_VERSION_2
,
68 DYNMEM_PROTOCOL_VERSION_CURRENT
= DYNMEM_PROTOCOL_VERSION_WIN8
77 enum dm_message_type
{
82 DM_VERSION_REQUEST
= 1,
83 DM_VERSION_RESPONSE
= 2,
84 DM_CAPABILITIES_REPORT
= 3,
85 DM_CAPABILITIES_RESPONSE
= 4,
87 DM_BALLOON_REQUEST
= 6,
88 DM_BALLOON_RESPONSE
= 7,
89 DM_UNBALLOON_REQUEST
= 8,
90 DM_UNBALLOON_RESPONSE
= 9,
91 DM_MEM_HOT_ADD_REQUEST
= 10,
92 DM_MEM_HOT_ADD_RESPONSE
= 11,
93 DM_VERSION_03_MAX
= 11,
103 * Structures defining the dynamic memory management
121 * To support guests that may have alignment
122 * limitations on hot-add, the guest can specify
123 * its alignment requirements; a value of n
124 * represents an alignment of 2^n in mega bytes.
126 __u64 hot_add_alignment
:4;
132 union dm_mem_page_range
{
135 * The PFN number of the first page in the range.
136 * 40 bits is the architectural limit of a PFN
141 * The number of pages in the range.
151 * The header for all dynamic memory messages:
153 * type: Type of the message.
154 * size: Size of the message in bytes; including the header.
155 * trans_id: The guest is responsible for manufacturing this ID.
165 * A generic message format for dynamic memory.
166 * Specific message formats are defined later in the file.
170 struct dm_header hdr
;
171 __u8 data
[]; /* enclosed message */
176 * Specific message types supporting the dynamic memory protocol.
180 * Version negotiation message. Sent from the guest to the host.
181 * The guest is free to try different versions until the host
182 * accepts the version.
184 * dm_version: The protocol version requested.
185 * is_last_attempt: If TRUE, this is the last version guest will request.
186 * reservedz: Reserved field, set to zero.
189 struct dm_version_request
{
190 struct dm_header hdr
;
191 union dm_version version
;
192 __u32 is_last_attempt
:1;
197 * Version response message; Host to Guest and indicates
198 * if the host has accepted the version sent by the guest.
200 * is_accepted: If TRUE, host has accepted the version and the guest
201 * should proceed to the next stage of the protocol. FALSE indicates that
202 * guest should re-try with a different version.
204 * reservedz: Reserved field, set to zero.
207 struct dm_version_response
{
208 struct dm_header hdr
;
214 * Message reporting capabilities. This is sent from the guest to the
218 struct dm_capabilities
{
219 struct dm_header hdr
;
222 __u64 max_page_number
;
226 * Response to the capabilities message. This is sent from the host to the
227 * guest. This message notifies if the host has accepted the guest's
228 * capabilities. If the host has not accepted, the guest must shutdown
231 * is_accepted: Indicates if the host has accepted guest's capabilities.
232 * reservedz: Must be 0.
235 struct dm_capabilities_resp_msg
{
236 struct dm_header hdr
;
242 * This message is used to report memory pressure from the guest.
243 * This message is not part of any transaction and there is no
244 * response to this message.
246 * num_avail: Available memory in pages.
247 * num_committed: Committed memory in pages.
248 * page_file_size: The accumulated size of all page files
249 * in the system in pages.
250 * zero_free: The nunber of zero and free pages.
251 * page_file_writes: The writes to the page file in pages.
252 * io_diff: An indicator of file cache efficiency or page file activity,
253 * calculated as File Cache Page Fault Count - Page Read Count.
254 * This value is in pages.
256 * Some of these metrics are Windows specific and fortunately
257 * the algorithm on the host side that computes the guest memory
258 * pressure only uses num_committed value.
262 struct dm_header hdr
;
265 __u64 page_file_size
;
267 __u32 page_file_writes
;
273 * Message to ask the guest to allocate memory - balloon up message.
274 * This message is sent from the host to the guest. The guest may not be
275 * able to allocate as much memory as requested.
277 * num_pages: number of pages to allocate.
281 struct dm_header hdr
;
288 * Balloon response message; this message is sent from the guest
289 * to the host in response to the balloon message.
291 * reservedz: Reserved; must be set to zero.
292 * more_pages: If FALSE, this is the last message of the transaction.
293 * if TRUE there will atleast one more message from the guest.
295 * range_count: The number of ranges in the range array.
297 * range_array: An array of page ranges returned to the host.
301 struct dm_balloon_response
{
302 struct dm_header hdr
;
305 __u32 range_count
:31;
306 union dm_mem_page_range range_array
[];
310 * Un-balloon message; this message is sent from the host
311 * to the guest to give guest more memory.
313 * more_pages: If FALSE, this is the last message of the transaction.
314 * if TRUE there will atleast one more message from the guest.
316 * reservedz: Reserved; must be set to zero.
318 * range_count: The number of ranges in the range array.
320 * range_array: An array of page ranges returned to the host.
324 struct dm_unballoon_request
{
325 struct dm_header hdr
;
329 union dm_mem_page_range range_array
[];
333 * Un-balloon response message; this message is sent from the guest
334 * to the host in response to an unballoon request.
338 struct dm_unballoon_response
{
339 struct dm_header hdr
;
344 * Hot add request message. Message sent from the host to the guest.
346 * mem_range: Memory range to hot add.
348 * On Linux we currently don't support this since we cannot hot add
349 * arbitrary granularity of memory.
353 struct dm_header hdr
;
354 union dm_mem_page_range range
;
358 * Hot add response message.
359 * This message is sent by the guest to report the status of a hot add request.
360 * If page_count is less than the requested page count, then the host should
361 * assume all further hot add requests will fail, since this indicates that
362 * the guest has hit an upper physical memory barrier.
364 * Hot adds may also fail due to low resources; in this case, the guest must
365 * not complete this message until the hot add can succeed, and the host must
366 * not send a new hot add request until the response is sent.
367 * If VSC fails to hot add memory DYNMEM_NUMBER_OF_UNSUCCESSFUL_HOTADD_ATTEMPTS
368 * times it fails the request.
371 * page_count: number of pages that were successfully hot added.
373 * result: result of the operation 1: success, 0: failure.
377 struct dm_hot_add_response
{
378 struct dm_header hdr
;
384 * Types of information sent from host to the guest.
388 INFO_TYPE_MAX_PAGE_CNT
= 0,
394 * Header for the information message.
397 struct dm_info_header
{
398 enum dm_info_type type
;
403 * This message is sent from the host to the guest to pass
404 * some relevant information (win8 addition).
407 * info_size: size of the information blob.
408 * info: information blob.
412 struct dm_header hdr
;
419 * End protocol definitions.
423 * State to manage hot adding memory into the guest.
424 * The range start_pfn : end_pfn specifies the range
425 * that the host has asked us to hot add. The range
426 * start_pfn : ha_end_pfn specifies the range that we have
427 * currently hot added. We hot add in multiples of 128M
428 * chunks; it is possible that we may not be able to bring
429 * online all the pages in the region. The range
430 * covered_start_pfn : covered_end_pfn defines the pages that can
434 struct hv_hotadd_state
{
435 struct list_head list
;
436 unsigned long start_pfn
;
437 unsigned long covered_start_pfn
;
438 unsigned long covered_end_pfn
;
439 unsigned long ha_end_pfn
;
440 unsigned long end_pfn
;
443 struct balloon_state
{
445 struct work_struct wrk
;
449 union dm_mem_page_range ha_page_range
;
450 union dm_mem_page_range ha_region_range
;
451 struct work_struct wrk
;
454 static bool hot_add
= true;
455 static bool do_hot_add
;
457 * Delay reporting memory pressure by
458 * the specified number of seconds.
460 static uint pressure_report_delay
= 45;
462 module_param(hot_add
, bool, (S_IRUGO
| S_IWUSR
));
463 MODULE_PARM_DESC(hot_add
, "If set attempt memory hot_add");
465 module_param(pressure_report_delay
, uint
, (S_IRUGO
| S_IWUSR
));
466 MODULE_PARM_DESC(pressure_report_delay
, "Delay in secs in reporting pressure");
467 static atomic_t trans_id
= ATOMIC_INIT(0);
469 static int dm_ring_size
= (5 * PAGE_SIZE
);
472 * Driver specific state.
485 static __u8 recv_buffer
[PAGE_SIZE
];
486 static __u8
*send_buffer
;
487 #define PAGES_IN_2M 512
488 #define HA_CHUNK (32 * 1024)
490 struct hv_dynmem_device
{
491 struct hv_device
*dev
;
492 enum hv_dm_state state
;
493 struct completion host_event
;
494 struct completion config_event
;
497 * Number of pages we have currently ballooned out.
499 unsigned int num_pages_ballooned
;
502 * State to manage the ballooning (up) operation.
504 struct balloon_state balloon_wrk
;
507 * State to execute the "hot-add" operation.
509 struct hot_add_wrk ha_wrk
;
512 * This state tracks if the host has specified a hot-add
515 bool host_specified_ha_region
;
518 * State to synchronize hot-add.
520 struct completion ol_waitevent
;
523 * This thread handles hot-add
524 * requests from the host as well as notifying
525 * the host with regards to memory pressure in
528 struct task_struct
*thread
;
531 * A list of hot-add regions.
533 struct list_head ha_region_list
;
536 * We start with the highest version we can support
537 * and downgrade based on the host; we save here the
538 * next version to try.
543 static struct hv_dynmem_device dm_device
;
545 #ifdef CONFIG_MEMORY_HOTPLUG
547 static void hv_bring_pgs_online(unsigned long start_pfn
, unsigned long size
)
551 for (i
= 0; i
< size
; i
++) {
553 pg
= pfn_to_page(start_pfn
+ i
);
554 __online_page_set_limits(pg
);
555 __online_page_increment_counters(pg
);
556 __online_page_free(pg
);
560 static void hv_mem_hot_add(unsigned long start
, unsigned long size
,
561 unsigned long pfn_count
,
562 struct hv_hotadd_state
*has
)
566 unsigned long start_pfn
;
567 unsigned long processed_pfn
;
568 unsigned long total_pfn
= pfn_count
;
570 for (i
= 0; i
< (size
/HA_CHUNK
); i
++) {
571 start_pfn
= start
+ (i
* HA_CHUNK
);
572 has
->ha_end_pfn
+= HA_CHUNK
;
574 if (total_pfn
> HA_CHUNK
) {
575 processed_pfn
= HA_CHUNK
;
576 total_pfn
-= HA_CHUNK
;
578 processed_pfn
= total_pfn
;
582 has
->covered_end_pfn
+= processed_pfn
;
584 init_completion(&dm_device
.ol_waitevent
);
585 dm_device
.ha_waiting
= true;
587 nid
= memory_add_physaddr_to_nid(PFN_PHYS(start_pfn
));
588 ret
= add_memory(nid
, PFN_PHYS((start_pfn
)),
589 (HA_CHUNK
<< PAGE_SHIFT
));
592 pr_info("hot_add memory failed error is %d\n", ret
);
593 if (ret
== -EEXIST
) {
595 * This error indicates that the error
596 * is not a transient failure. This is the
597 * case where the guest's physical address map
598 * precludes hot adding memory. Stop all further
603 has
->ha_end_pfn
-= HA_CHUNK
;
604 has
->covered_end_pfn
-= processed_pfn
;
609 * Wait for the memory block to be onlined.
611 t
= wait_for_completion_timeout(&dm_device
.ol_waitevent
, 5*HZ
);
613 pr_info("hot_add memory timedout\n");
614 has
->ha_end_pfn
-= HA_CHUNK
;
615 has
->covered_end_pfn
-= processed_pfn
;
624 static void hv_online_page(struct page
*pg
)
626 struct list_head
*cur
;
627 struct hv_hotadd_state
*has
;
628 unsigned long cur_start_pgp
;
629 unsigned long cur_end_pgp
;
631 if (dm_device
.ha_waiting
) {
632 dm_device
.ha_waiting
= false;
633 complete(&dm_device
.ol_waitevent
);
636 list_for_each(cur
, &dm_device
.ha_region_list
) {
637 has
= list_entry(cur
, struct hv_hotadd_state
, list
);
638 cur_start_pgp
= (unsigned long)
639 pfn_to_page(has
->covered_start_pfn
);
640 cur_end_pgp
= (unsigned long)pfn_to_page(has
->covered_end_pfn
);
642 if (((unsigned long)pg
>= cur_start_pgp
) &&
643 ((unsigned long)pg
< cur_end_pgp
)) {
645 * This frame is currently backed; online the
648 __online_page_set_limits(pg
);
649 __online_page_increment_counters(pg
);
650 __online_page_free(pg
);
651 has
->covered_start_pfn
++;
656 static bool pfn_covered(unsigned long start_pfn
, unsigned long pfn_cnt
)
658 struct list_head
*cur
;
659 struct hv_hotadd_state
*has
;
660 unsigned long residual
, new_inc
;
662 if (list_empty(&dm_device
.ha_region_list
))
665 list_for_each(cur
, &dm_device
.ha_region_list
) {
666 has
= list_entry(cur
, struct hv_hotadd_state
, list
);
669 * If the pfn range we are dealing with is not in the current
670 * "hot add block", move on.
672 if ((start_pfn
>= has
->end_pfn
))
675 * If the current hot add-request extends beyond
676 * our current limit; extend it.
678 if ((start_pfn
+ pfn_cnt
) > has
->end_pfn
) {
679 residual
= (start_pfn
+ pfn_cnt
- has
->end_pfn
);
681 * Extend the region by multiples of HA_CHUNK.
683 new_inc
= (residual
/ HA_CHUNK
) * HA_CHUNK
;
684 if (residual
% HA_CHUNK
)
687 has
->end_pfn
+= new_inc
;
691 * If the current start pfn is not where the covered_end
695 if (has
->covered_end_pfn
!= start_pfn
) {
696 has
->covered_end_pfn
= start_pfn
;
697 has
->covered_start_pfn
= start_pfn
;
706 static unsigned long handle_pg_range(unsigned long pg_start
,
707 unsigned long pg_count
)
709 unsigned long start_pfn
= pg_start
;
710 unsigned long pfn_cnt
= pg_count
;
712 struct list_head
*cur
;
713 struct hv_hotadd_state
*has
;
714 unsigned long pgs_ol
= 0;
715 unsigned long old_covered_state
;
717 if (list_empty(&dm_device
.ha_region_list
))
720 list_for_each(cur
, &dm_device
.ha_region_list
) {
721 has
= list_entry(cur
, struct hv_hotadd_state
, list
);
724 * If the pfn range we are dealing with is not in the current
725 * "hot add block", move on.
727 if ((start_pfn
>= has
->end_pfn
))
730 old_covered_state
= has
->covered_end_pfn
;
732 if (start_pfn
< has
->ha_end_pfn
) {
734 * This is the case where we are backing pages
735 * in an already hot added region. Bring
736 * these pages online first.
738 pgs_ol
= has
->ha_end_pfn
- start_pfn
;
739 if (pgs_ol
> pfn_cnt
)
741 hv_bring_pgs_online(start_pfn
, pgs_ol
);
742 has
->covered_end_pfn
+= pgs_ol
;
743 has
->covered_start_pfn
+= pgs_ol
;
747 if ((has
->ha_end_pfn
< has
->end_pfn
) && (pfn_cnt
> 0)) {
749 * We have some residual hot add range
750 * that needs to be hot added; hot add
751 * it now. Hot add a multiple of
752 * of HA_CHUNK that fully covers the pages
755 size
= (has
->end_pfn
- has
->ha_end_pfn
);
756 if (pfn_cnt
<= size
) {
757 size
= ((pfn_cnt
/ HA_CHUNK
) * HA_CHUNK
);
758 if (pfn_cnt
% HA_CHUNK
)
763 hv_mem_hot_add(has
->ha_end_pfn
, size
, pfn_cnt
, has
);
766 * If we managed to online any pages that were given to us,
767 * we declare success.
769 return has
->covered_end_pfn
- old_covered_state
;
776 static unsigned long process_hot_add(unsigned long pg_start
,
777 unsigned long pfn_cnt
,
778 unsigned long rg_start
,
779 unsigned long rg_size
)
781 struct hv_hotadd_state
*ha_region
= NULL
;
786 if (!dm_device
.host_specified_ha_region
)
787 if (pfn_covered(pg_start
, pfn_cnt
))
791 * If the host has specified a hot-add range; deal with it first.
795 ha_region
= kzalloc(sizeof(struct hv_hotadd_state
), GFP_KERNEL
);
799 INIT_LIST_HEAD(&ha_region
->list
);
801 list_add_tail(&ha_region
->list
, &dm_device
.ha_region_list
);
802 ha_region
->start_pfn
= rg_start
;
803 ha_region
->ha_end_pfn
= rg_start
;
804 ha_region
->covered_start_pfn
= pg_start
;
805 ha_region
->covered_end_pfn
= pg_start
;
806 ha_region
->end_pfn
= rg_start
+ rg_size
;
811 * Process the page range specified; bringing them
812 * online if possible.
814 return handle_pg_range(pg_start
, pfn_cnt
);
819 static void hot_add_req(struct work_struct
*dummy
)
821 struct dm_hot_add_response resp
;
822 #ifdef CONFIG_MEMORY_HOTPLUG
823 unsigned long pg_start
, pfn_cnt
;
824 unsigned long rg_start
, rg_sz
;
826 struct hv_dynmem_device
*dm
= &dm_device
;
828 memset(&resp
, 0, sizeof(struct dm_hot_add_response
));
829 resp
.hdr
.type
= DM_MEM_HOT_ADD_RESPONSE
;
830 resp
.hdr
.size
= sizeof(struct dm_hot_add_response
);
831 resp
.hdr
.trans_id
= atomic_inc_return(&trans_id
);
833 #ifdef CONFIG_MEMORY_HOTPLUG
834 pg_start
= dm
->ha_wrk
.ha_page_range
.finfo
.start_page
;
835 pfn_cnt
= dm
->ha_wrk
.ha_page_range
.finfo
.page_cnt
;
837 rg_start
= dm
->ha_wrk
.ha_region_range
.finfo
.start_page
;
838 rg_sz
= dm
->ha_wrk
.ha_region_range
.finfo
.page_cnt
;
840 if ((rg_start
== 0) && (!dm
->host_specified_ha_region
)) {
841 unsigned long region_size
;
842 unsigned long region_start
;
845 * The host has not specified the hot-add region.
846 * Based on the hot-add page range being specified,
847 * compute a hot-add region that can cover the pages
848 * that need to be hot-added while ensuring the alignment
849 * and size requirements of Linux as it relates to hot-add.
851 region_start
= pg_start
;
852 region_size
= (pfn_cnt
/ HA_CHUNK
) * HA_CHUNK
;
853 if (pfn_cnt
% HA_CHUNK
)
854 region_size
+= HA_CHUNK
;
856 region_start
= (pg_start
/ HA_CHUNK
) * HA_CHUNK
;
858 rg_start
= region_start
;
863 resp
.page_count
= process_hot_add(pg_start
, pfn_cnt
,
867 * The result field of the response structure has the
868 * following semantics:
870 * 1. If all or some pages hot-added: Guest should return success.
872 * 2. If no pages could be hot-added:
874 * If the guest returns success, then the host
875 * will not attempt any further hot-add operations. This
876 * signifies a permanent failure.
878 * If the guest returns failure, then this failure will be
879 * treated as a transient failure and the host may retry the
880 * hot-add operation after some delay.
882 if (resp
.page_count
> 0)
884 else if (!do_hot_add
)
889 if (!do_hot_add
|| (resp
.page_count
== 0))
890 pr_info("Memory hot add failed\n");
892 dm
->state
= DM_INITIALIZED
;
893 vmbus_sendpacket(dm
->dev
->channel
, &resp
,
894 sizeof(struct dm_hot_add_response
),
896 VM_PKT_DATA_INBAND
, 0);
899 static void process_info(struct hv_dynmem_device
*dm
, struct dm_info_msg
*msg
)
901 struct dm_info_header
*info_hdr
;
903 info_hdr
= (struct dm_info_header
*)msg
->info
;
905 switch (info_hdr
->type
) {
906 case INFO_TYPE_MAX_PAGE_CNT
:
907 pr_info("Received INFO_TYPE_MAX_PAGE_CNT\n");
908 pr_info("Data Size is %d\n", info_hdr
->data_size
);
911 pr_info("Received Unknown type: %d\n", info_hdr
->type
);
915 static unsigned long compute_balloon_floor(void)
917 unsigned long min_pages
;
918 #define MB2PAGES(mb) ((mb) << (20 - PAGE_SHIFT))
919 /* Simple continuous piecewiese linear function:
920 * max MiB -> min MiB gradient
931 if (totalram_pages
< MB2PAGES(128))
932 min_pages
= MB2PAGES(8) + (totalram_pages
>> 1);
933 else if (totalram_pages
< MB2PAGES(512))
934 min_pages
= MB2PAGES(40) + (totalram_pages
>> 2);
935 else if (totalram_pages
< MB2PAGES(2048))
936 min_pages
= MB2PAGES(104) + (totalram_pages
>> 3);
938 min_pages
= MB2PAGES(296) + (totalram_pages
>> 5);
944 * Post our status as it relates memory pressure to the
945 * host. Host expects the guests to post this status
946 * periodically at 1 second intervals.
948 * The metrics specified in this protocol are very Windows
949 * specific and so we cook up numbers here to convey our memory
953 static void post_status(struct hv_dynmem_device
*dm
)
955 struct dm_status status
;
958 if (pressure_report_delay
> 0) {
959 --pressure_report_delay
;
963 memset(&status
, 0, sizeof(struct dm_status
));
964 status
.hdr
.type
= DM_STATUS_REPORT
;
965 status
.hdr
.size
= sizeof(struct dm_status
);
966 status
.hdr
.trans_id
= atomic_inc_return(&trans_id
);
969 * The host expects the guest to report free memory.
970 * Further, the host expects the pressure information to
971 * include the ballooned out pages.
972 * For a given amount of memory that we are managing, we
973 * need to compute a floor below which we should not balloon.
974 * Compute this and add it to the pressure report.
976 status
.num_avail
= val
.freeram
;
977 status
.num_committed
= vm_memory_committed() +
978 dm
->num_pages_ballooned
+
979 compute_balloon_floor();
981 vmbus_sendpacket(dm
->dev
->channel
, &status
,
982 sizeof(struct dm_status
),
984 VM_PKT_DATA_INBAND
, 0);
988 static void free_balloon_pages(struct hv_dynmem_device
*dm
,
989 union dm_mem_page_range
*range_array
)
991 int num_pages
= range_array
->finfo
.page_cnt
;
992 __u64 start_frame
= range_array
->finfo
.start_page
;
996 for (i
= 0; i
< num_pages
; i
++) {
997 pg
= pfn_to_page(i
+ start_frame
);
999 dm
->num_pages_ballooned
--;
1005 static int alloc_balloon_pages(struct hv_dynmem_device
*dm
, int num_pages
,
1006 struct dm_balloon_response
*bl_resp
, int alloc_unit
,
1012 if (num_pages
< alloc_unit
)
1015 for (i
= 0; (i
* alloc_unit
) < num_pages
; i
++) {
1016 if (bl_resp
->hdr
.size
+ sizeof(union dm_mem_page_range
) >
1018 return i
* alloc_unit
;
1021 * We execute this code in a thread context. Furthermore,
1022 * we don't want the kernel to try too hard.
1024 pg
= alloc_pages(GFP_HIGHUSER
| __GFP_NORETRY
|
1025 __GFP_NOMEMALLOC
| __GFP_NOWARN
,
1026 get_order(alloc_unit
<< PAGE_SHIFT
));
1029 *alloc_error
= true;
1030 return i
* alloc_unit
;
1034 dm
->num_pages_ballooned
+= alloc_unit
;
1037 * If we allocatted 2M pages; split them so we
1038 * can free them in any order we get.
1041 if (alloc_unit
!= 1)
1042 split_page(pg
, get_order(alloc_unit
<< PAGE_SHIFT
));
1044 bl_resp
->range_count
++;
1045 bl_resp
->range_array
[i
].finfo
.start_page
=
1047 bl_resp
->range_array
[i
].finfo
.page_cnt
= alloc_unit
;
1048 bl_resp
->hdr
.size
+= sizeof(union dm_mem_page_range
);
1057 static void balloon_up(struct work_struct
*dummy
)
1059 int num_pages
= dm_device
.balloon_wrk
.num_pages
;
1060 int num_ballooned
= 0;
1061 struct dm_balloon_response
*bl_resp
;
1064 bool alloc_error
= false;
1070 * We will attempt 2M allocations. However, if we fail to
1071 * allocate 2M chunks, we will go back to 4k allocations.
1076 bl_resp
= (struct dm_balloon_response
*)send_buffer
;
1077 memset(send_buffer
, 0, PAGE_SIZE
);
1078 bl_resp
->hdr
.type
= DM_BALLOON_RESPONSE
;
1079 bl_resp
->hdr
.trans_id
= atomic_inc_return(&trans_id
);
1080 bl_resp
->hdr
.size
= sizeof(struct dm_balloon_response
);
1081 bl_resp
->more_pages
= 1;
1084 num_pages
-= num_ballooned
;
1085 num_ballooned
= alloc_balloon_pages(&dm_device
, num_pages
,
1086 bl_resp
, alloc_unit
,
1089 if ((alloc_error
) && (alloc_unit
!= 1)) {
1094 if ((alloc_error
) || (num_ballooned
== num_pages
)) {
1095 bl_resp
->more_pages
= 0;
1097 dm_device
.state
= DM_INITIALIZED
;
1101 * We are pushing a lot of data through the channel;
1102 * deal with transient failures caused because of the
1103 * lack of space in the ring buffer.
1107 ret
= vmbus_sendpacket(dm_device
.dev
->channel
,
1110 (unsigned long)NULL
,
1111 VM_PKT_DATA_INBAND
, 0);
1116 } while (ret
== -EAGAIN
);
1120 * Free up the memory we allocatted.
1122 pr_info("Balloon response failed\n");
1124 for (i
= 0; i
< bl_resp
->range_count
; i
++)
1125 free_balloon_pages(&dm_device
,
1126 &bl_resp
->range_array
[i
]);
1134 static void balloon_down(struct hv_dynmem_device
*dm
,
1135 struct dm_unballoon_request
*req
)
1137 union dm_mem_page_range
*range_array
= req
->range_array
;
1138 int range_count
= req
->range_count
;
1139 struct dm_unballoon_response resp
;
1142 for (i
= 0; i
< range_count
; i
++)
1143 free_balloon_pages(dm
, &range_array
[i
]);
1145 if (req
->more_pages
== 1)
1148 memset(&resp
, 0, sizeof(struct dm_unballoon_response
));
1149 resp
.hdr
.type
= DM_UNBALLOON_RESPONSE
;
1150 resp
.hdr
.trans_id
= atomic_inc_return(&trans_id
);
1151 resp
.hdr
.size
= sizeof(struct dm_unballoon_response
);
1153 vmbus_sendpacket(dm_device
.dev
->channel
, &resp
,
1154 sizeof(struct dm_unballoon_response
),
1155 (unsigned long)NULL
,
1156 VM_PKT_DATA_INBAND
, 0);
1158 dm
->state
= DM_INITIALIZED
;
1161 static void balloon_onchannelcallback(void *context
);
1163 static int dm_thread_func(void *dm_dev
)
1165 struct hv_dynmem_device
*dm
= dm_dev
;
1168 while (!kthread_should_stop()) {
1169 t
= wait_for_completion_timeout(&dm_device
.config_event
, 1*HZ
);
1171 * The host expects us to post information on the memory
1172 * pressure every second.
1184 static void version_resp(struct hv_dynmem_device
*dm
,
1185 struct dm_version_response
*vresp
)
1187 struct dm_version_request version_req
;
1190 if (vresp
->is_accepted
) {
1192 * We are done; wakeup the
1193 * context waiting for version
1196 complete(&dm
->host_event
);
1200 * If there are more versions to try, continue
1201 * with negotiations; if not
1202 * shutdown the service since we are not able
1203 * to negotiate a suitable version number
1206 if (dm
->next_version
== 0)
1209 dm
->next_version
= 0;
1210 memset(&version_req
, 0, sizeof(struct dm_version_request
));
1211 version_req
.hdr
.type
= DM_VERSION_REQUEST
;
1212 version_req
.hdr
.size
= sizeof(struct dm_version_request
);
1213 version_req
.hdr
.trans_id
= atomic_inc_return(&trans_id
);
1214 version_req
.version
.version
= DYNMEM_PROTOCOL_VERSION_WIN7
;
1215 version_req
.is_last_attempt
= 1;
1217 ret
= vmbus_sendpacket(dm
->dev
->channel
, &version_req
,
1218 sizeof(struct dm_version_request
),
1219 (unsigned long)NULL
,
1220 VM_PKT_DATA_INBAND
, 0);
1228 dm
->state
= DM_INIT_ERROR
;
1229 complete(&dm
->host_event
);
1232 static void cap_resp(struct hv_dynmem_device
*dm
,
1233 struct dm_capabilities_resp_msg
*cap_resp
)
1235 if (!cap_resp
->is_accepted
) {
1236 pr_info("Capabilities not accepted by host\n");
1237 dm
->state
= DM_INIT_ERROR
;
1239 complete(&dm
->host_event
);
1242 static void balloon_onchannelcallback(void *context
)
1244 struct hv_device
*dev
= context
;
1247 struct dm_message
*dm_msg
;
1248 struct dm_header
*dm_hdr
;
1249 struct hv_dynmem_device
*dm
= hv_get_drvdata(dev
);
1250 struct dm_balloon
*bal_msg
;
1251 struct dm_hot_add
*ha_msg
;
1252 union dm_mem_page_range
*ha_pg_range
;
1253 union dm_mem_page_range
*ha_region
;
1255 memset(recv_buffer
, 0, sizeof(recv_buffer
));
1256 vmbus_recvpacket(dev
->channel
, recv_buffer
,
1257 PAGE_SIZE
, &recvlen
, &requestid
);
1260 dm_msg
= (struct dm_message
*)recv_buffer
;
1261 dm_hdr
= &dm_msg
->hdr
;
1263 switch (dm_hdr
->type
) {
1264 case DM_VERSION_RESPONSE
:
1266 (struct dm_version_response
*)dm_msg
);
1269 case DM_CAPABILITIES_RESPONSE
:
1271 (struct dm_capabilities_resp_msg
*)dm_msg
);
1274 case DM_BALLOON_REQUEST
:
1275 if (dm
->state
== DM_BALLOON_UP
)
1276 pr_warn("Currently ballooning\n");
1277 bal_msg
= (struct dm_balloon
*)recv_buffer
;
1278 dm
->state
= DM_BALLOON_UP
;
1279 dm_device
.balloon_wrk
.num_pages
= bal_msg
->num_pages
;
1280 schedule_work(&dm_device
.balloon_wrk
.wrk
);
1283 case DM_UNBALLOON_REQUEST
:
1284 dm
->state
= DM_BALLOON_DOWN
;
1286 (struct dm_unballoon_request
*)recv_buffer
);
1289 case DM_MEM_HOT_ADD_REQUEST
:
1290 if (dm
->state
== DM_HOT_ADD
)
1291 pr_warn("Currently hot-adding\n");
1292 dm
->state
= DM_HOT_ADD
;
1293 ha_msg
= (struct dm_hot_add
*)recv_buffer
;
1294 if (ha_msg
->hdr
.size
== sizeof(struct dm_hot_add
)) {
1296 * This is a normal hot-add request specifying
1299 ha_pg_range
= &ha_msg
->range
;
1300 dm
->ha_wrk
.ha_page_range
= *ha_pg_range
;
1301 dm
->ha_wrk
.ha_region_range
.page_range
= 0;
1304 * Host is specifying that we first hot-add
1305 * a region and then partially populate this
1308 dm
->host_specified_ha_region
= true;
1309 ha_pg_range
= &ha_msg
->range
;
1310 ha_region
= &ha_pg_range
[1];
1311 dm
->ha_wrk
.ha_page_range
= *ha_pg_range
;
1312 dm
->ha_wrk
.ha_region_range
= *ha_region
;
1314 schedule_work(&dm_device
.ha_wrk
.wrk
);
1317 case DM_INFO_MESSAGE
:
1318 process_info(dm
, (struct dm_info_msg
*)dm_msg
);
1322 pr_err("Unhandled message: type: %d\n", dm_hdr
->type
);
1329 static int balloon_probe(struct hv_device
*dev
,
1330 const struct hv_vmbus_device_id
*dev_id
)
1333 struct dm_version_request version_req
;
1334 struct dm_capabilities cap_msg
;
1336 do_hot_add
= hot_add
;
1339 * First allocate a send buffer.
1342 send_buffer
= kmalloc(PAGE_SIZE
, GFP_KERNEL
);
1346 ret
= vmbus_open(dev
->channel
, dm_ring_size
, dm_ring_size
, NULL
, 0,
1347 balloon_onchannelcallback
, dev
);
1352 dm_device
.dev
= dev
;
1353 dm_device
.state
= DM_INITIALIZING
;
1354 dm_device
.next_version
= DYNMEM_PROTOCOL_VERSION_WIN7
;
1355 init_completion(&dm_device
.host_event
);
1356 init_completion(&dm_device
.config_event
);
1357 INIT_LIST_HEAD(&dm_device
.ha_region_list
);
1358 INIT_WORK(&dm_device
.balloon_wrk
.wrk
, balloon_up
);
1359 INIT_WORK(&dm_device
.ha_wrk
.wrk
, hot_add_req
);
1360 dm_device
.host_specified_ha_region
= false;
1363 kthread_run(dm_thread_func
, &dm_device
, "hv_balloon");
1364 if (IS_ERR(dm_device
.thread
)) {
1365 ret
= PTR_ERR(dm_device
.thread
);
1369 #ifdef CONFIG_MEMORY_HOTPLUG
1370 set_online_page_callback(&hv_online_page
);
1373 hv_set_drvdata(dev
, &dm_device
);
1375 * Initiate the hand shake with the host and negotiate
1376 * a version that the host can support. We start with the
1377 * highest version number and go down if the host cannot
1380 memset(&version_req
, 0, sizeof(struct dm_version_request
));
1381 version_req
.hdr
.type
= DM_VERSION_REQUEST
;
1382 version_req
.hdr
.size
= sizeof(struct dm_version_request
);
1383 version_req
.hdr
.trans_id
= atomic_inc_return(&trans_id
);
1384 version_req
.version
.version
= DYNMEM_PROTOCOL_VERSION_WIN8
;
1385 version_req
.is_last_attempt
= 0;
1387 ret
= vmbus_sendpacket(dev
->channel
, &version_req
,
1388 sizeof(struct dm_version_request
),
1389 (unsigned long)NULL
,
1390 VM_PKT_DATA_INBAND
, 0);
1394 t
= wait_for_completion_timeout(&dm_device
.host_event
, 5*HZ
);
1401 * If we could not negotiate a compatible version with the host
1402 * fail the probe function.
1404 if (dm_device
.state
== DM_INIT_ERROR
) {
1409 * Now submit our capabilities to the host.
1411 memset(&cap_msg
, 0, sizeof(struct dm_capabilities
));
1412 cap_msg
.hdr
.type
= DM_CAPABILITIES_REPORT
;
1413 cap_msg
.hdr
.size
= sizeof(struct dm_capabilities
);
1414 cap_msg
.hdr
.trans_id
= atomic_inc_return(&trans_id
);
1416 cap_msg
.caps
.cap_bits
.balloon
= 1;
1417 cap_msg
.caps
.cap_bits
.hot_add
= 1;
1420 * Specify our alignment requirements as it relates
1421 * memory hot-add. Specify 128MB alignment.
1423 cap_msg
.caps
.cap_bits
.hot_add_alignment
= 7;
1426 * Currently the host does not use these
1427 * values and we set them to what is done in the
1430 cap_msg
.min_page_cnt
= 0;
1431 cap_msg
.max_page_number
= -1;
1433 ret
= vmbus_sendpacket(dev
->channel
, &cap_msg
,
1434 sizeof(struct dm_capabilities
),
1435 (unsigned long)NULL
,
1436 VM_PKT_DATA_INBAND
, 0);
1440 t
= wait_for_completion_timeout(&dm_device
.host_event
, 5*HZ
);
1447 * If the host does not like our capabilities,
1448 * fail the probe function.
1450 if (dm_device
.state
== DM_INIT_ERROR
) {
1455 dm_device
.state
= DM_INITIALIZED
;
1460 #ifdef CONFIG_MEMORY_HOTPLUG
1461 restore_online_page_callback(&hv_online_page
);
1463 kthread_stop(dm_device
.thread
);
1466 vmbus_close(dev
->channel
);
1472 static int balloon_remove(struct hv_device
*dev
)
1474 struct hv_dynmem_device
*dm
= hv_get_drvdata(dev
);
1475 struct list_head
*cur
, *tmp
;
1476 struct hv_hotadd_state
*has
;
1478 if (dm
->num_pages_ballooned
!= 0)
1479 pr_warn("Ballooned pages: %d\n", dm
->num_pages_ballooned
);
1481 cancel_work_sync(&dm
->balloon_wrk
.wrk
);
1482 cancel_work_sync(&dm
->ha_wrk
.wrk
);
1484 vmbus_close(dev
->channel
);
1485 kthread_stop(dm
->thread
);
1487 #ifdef CONFIG_MEMORY_HOTPLUG
1488 restore_online_page_callback(&hv_online_page
);
1490 list_for_each_safe(cur
, tmp
, &dm
->ha_region_list
) {
1491 has
= list_entry(cur
, struct hv_hotadd_state
, list
);
1492 list_del(&has
->list
);
1499 static const struct hv_vmbus_device_id id_table
[] = {
1500 /* Dynamic Memory Class ID */
1501 /* 525074DC-8985-46e2-8057-A307DC18A502 */
1506 MODULE_DEVICE_TABLE(vmbus
, id_table
);
1508 static struct hv_driver balloon_drv
= {
1509 .name
= "hv_balloon",
1510 .id_table
= id_table
,
1511 .probe
= balloon_probe
,
1512 .remove
= balloon_remove
,
1515 static int __init
init_balloon_drv(void)
1518 return vmbus_driver_register(&balloon_drv
);
1521 module_init(init_balloon_drv
);
1523 MODULE_DESCRIPTION("Hyper-V Balloon");
1524 MODULE_VERSION(HV_DRV_VERSION
);
1525 MODULE_LICENSE("GPL");