diff --git a/arch/x86/kvm/svm/sev.c b/arch/x86/kvm/svm/sev.c index 9e9b8ca2e9f7..2d0f5a802ae1 100644 --- a/arch/x86/kvm/svm/sev.c +++ b/arch/x86/kvm/svm/sev.c @@ -720,14 +720,50 @@ static int sev_launch_start(struct kvm *kvm, struct kvm_sev_cmd *argp) return ret; } +static int sev_check_pin_count(struct kvm *kvm, unsigned long npages) +{ + unsigned long total_npages, lock_limit; + + total_npages = to_kvm_sev_info(kvm)->pages_locked + npages; + if (total_npages > totalram_pages()) + return -EINVAL; + + lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; + if (total_npages > lock_limit && !capable(CAP_IPC_LOCK)) { + pr_err_ratelimited("SEV: %lu total pages would exceed the lock limit of %lu.\n", + total_npages, lock_limit); + return -ENOMEM; + } + + return 0; +} + +static int sev_pin_user_pages(struct kvm *kvm, unsigned long addr, int npages, + unsigned int gup_flags, struct page **pages) +{ + int npinned; + + lockdep_assert_held(&kvm->lock); + + npinned = pin_user_pages_fast(addr, npages, gup_flags, pages); + if (npinned != npages) { + if (npinned > 0) + unpin_user_pages(pages, npinned); + pr_err_ratelimited("SEV: Failure locking %u pages.\n", npages); + return -ENOMEM; + } + + to_kvm_sev_info(kvm)->pages_locked += npages; + return 0; +} + static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, unsigned long ulen, unsigned long *n, unsigned int flags) { - struct kvm_sev_info *sev = to_kvm_sev_info(kvm); - unsigned long npages, total_npages, lock_limit; + unsigned long npages; struct page **pages; - int npinned, ret; + int ret; lockdep_assert_held(&kvm->lock); @@ -743,16 +779,9 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, if (npages > INT_MAX) return ERR_PTR(-EINVAL); - total_npages = sev->pages_locked + npages; - if (total_npages > totalram_pages()) - return ERR_PTR(-EINVAL); - - lock_limit = rlimit(RLIMIT_MEMLOCK) >> PAGE_SHIFT; - if (total_npages > lock_limit && !capable(CAP_IPC_LOCK)) { - pr_err("SEV: %lu total pages would exceed the lock limit of %lu.\n", - total_npages, lock_limit); - return ERR_PTR(-ENOMEM); - } + ret = sev_check_pin_count(kvm, npages); + if (ret) + return ERR_PTR(ret); /* * Don't WARN if the kernel (rightly) thinks the total size is absurd, @@ -764,25 +793,14 @@ static struct page **sev_pin_memory(struct kvm *kvm, unsigned long uaddr, if (!pages) return ERR_PTR(-ENOMEM); - /* Pin the user virtual address. */ - npinned = pin_user_pages_fast(uaddr, npages, flags, pages); - if (npinned != npages) { - pr_err("SEV: Failure locking %lu pages.\n", npages); - ret = -ENOMEM; - goto err; + ret = sev_pin_user_pages(kvm, uaddr, npages, flags, pages); + if (ret) { + kvfree(pages); + return ERR_PTR(ret); } *n = npages; - sev->pages_locked = total_npages; - return pages; - -err: - if (npinned > 0) - unpin_user_pages(pages, npinned); - - kvfree(pages); - return ERR_PTR(ret); } static void sev_unpin_memory(struct kvm *kvm, struct page **pages, @@ -793,6 +811,29 @@ static void sev_unpin_memory(struct kvm *kvm, struct page **pages, to_kvm_sev_info(kvm)->pages_locked -= npages; } +static struct page *sev_pin_page(struct kvm *kvm, unsigned long addr, + unsigned int flags) +{ + struct page *page; + int r; + + r = sev_check_pin_count(kvm, 1); + if (r) + return ERR_PTR(r); + + r = sev_pin_user_pages(kvm, addr, 1, flags, &page); + if (r) + return ERR_PTR(r); + + return page; +} + +static void sev_unpin_page(struct kvm *kvm, struct page *page) +{ + unpin_user_pages(&page, 1); + to_kvm_sev_info(kvm)->pages_locked -= 1; +} + static void sev_clflush_pages(struct page *pages[], unsigned long npages) { uint8_t *page_virtual; @@ -1345,9 +1386,8 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) { unsigned long vaddr, vaddr_end, next_vaddr; unsigned long dst_vaddr; - struct page **src_p, **dst_p; + struct page *src_p, *dst_p; struct kvm_sev_dbg debug; - unsigned long n; unsigned int size; int ret; @@ -1373,13 +1413,13 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) int len, s_off, d_off; /* lock userspace source and destination page */ - src_p = sev_pin_memory(kvm, vaddr & PAGE_MASK, PAGE_SIZE, &n, 0); + src_p = sev_pin_page(kvm, vaddr & PAGE_MASK, 0); if (IS_ERR(src_p)) return PTR_ERR(src_p); - dst_p = sev_pin_memory(kvm, dst_vaddr & PAGE_MASK, PAGE_SIZE, &n, FOLL_WRITE); + dst_p = sev_pin_page(kvm, dst_vaddr & PAGE_MASK, FOLL_WRITE); if (IS_ERR(dst_p)) { - sev_unpin_memory(kvm, src_p, n); + sev_unpin_page(kvm, src_p); return PTR_ERR(dst_p); } @@ -1388,8 +1428,8 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) * the pages; flush the destination too so that future accesses do not * see stale data. */ - sev_clflush_pages(src_p, 1); - sev_clflush_pages(dst_p, 1); + sev_clflush_pages(&src_p, 1); + sev_clflush_pages(&dst_p, 1); /* * Since user buffer may not be page aligned, calculate the @@ -1402,20 +1442,20 @@ static int sev_dbg_crypt(struct kvm *kvm, struct kvm_sev_cmd *argp, bool dec) if (dec) ret = __sev_dbg_decrypt_user(kvm, - __sme_page_pa(src_p[0]) + s_off, + __sme_page_pa(src_p) + s_off, (void __user *)dst_vaddr, - __sme_page_pa(dst_p[0]) + d_off, + __sme_page_pa(dst_p) + d_off, len, &argp->error); else ret = __sev_dbg_encrypt_user(kvm, - __sme_page_pa(src_p[0]) + s_off, + __sme_page_pa(src_p) + s_off, (void __user *)vaddr, - __sme_page_pa(dst_p[0]) + d_off, + __sme_page_pa(dst_p) + d_off, (void __user *)dst_vaddr, len, &argp->error); - sev_unpin_memory(kvm, src_p, n); - sev_unpin_memory(kvm, dst_p, n); + sev_unpin_page(kvm, src_p); + sev_unpin_page(kvm, dst_p); if (ret) goto err; @@ -1698,8 +1738,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) struct sev_data_send_update_data data; struct kvm_sev_send_update_data params; void *hdr, *trans_data; - struct page **guest_page; - unsigned long n; + struct page *guest_page; int ret, offset; if (!sev_guest(kvm)) @@ -1723,8 +1762,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) return -EINVAL; /* Pin guest memory */ - guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK, - PAGE_SIZE, &n, 0); + guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, 0); if (IS_ERR(guest_page)) return PTR_ERR(guest_page); @@ -1745,7 +1783,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) data.trans_len = params.trans_len; /* The SEND_UPDATE_DATA command requires C-bit to be always set. */ - data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset; + data.guest_address = page_to_phys(guest_page) + offset; data.guest_address |= sev_me_mask; data.guest_len = params.guest_len; data.handle = to_kvm_sev_info(kvm)->handle; @@ -1772,8 +1810,7 @@ static int sev_send_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) e_free_hdr: kfree(hdr); e_unpin: - sev_unpin_memory(kvm, guest_page, n); - + sev_unpin_page(kvm, guest_page); return ret; } @@ -1878,8 +1915,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) struct kvm_sev_receive_update_data params; struct sev_data_receive_update_data data; void *hdr = NULL, *trans = NULL; - struct page **guest_page; - unsigned long n; + struct page *guest_page; int ret, offset; if (!sev_guest(kvm)) @@ -1916,8 +1952,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) data.trans_len = params.trans_len; /* Pin guest memory */ - guest_page = sev_pin_memory(kvm, params.guest_uaddr & PAGE_MASK, - PAGE_SIZE, &n, FOLL_WRITE); + guest_page = sev_pin_page(kvm, params.guest_uaddr & PAGE_MASK, FOLL_WRITE); if (IS_ERR(guest_page)) { ret = PTR_ERR(guest_page); goto e_free_trans; @@ -1928,10 +1963,10 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) * encrypts the written data with the guest's key, and the cache may * contain dirty, unencrypted data. */ - sev_clflush_pages(guest_page, n); + sev_clflush_pages(&guest_page, 1); /* The RECEIVE_UPDATE_DATA command requires C-bit to be always set. */ - data.guest_address = (page_to_pfn(guest_page[0]) << PAGE_SHIFT) + offset; + data.guest_address = page_to_phys(guest_page) + offset; data.guest_address |= sev_me_mask; data.guest_len = params.guest_len; data.handle = to_kvm_sev_info(kvm)->handle; @@ -1939,7 +1974,7 @@ static int sev_receive_update_data(struct kvm *kvm, struct kvm_sev_cmd *argp) ret = sev_issue_cmd(kvm, SEV_CMD_RECEIVE_UPDATE_DATA, &data, &argp->error); - sev_unpin_memory(kvm, guest_page, n); + sev_unpin_page(kvm, guest_page); e_free_trans: kfree(trans);