1970 lines
50 KiB
C
1970 lines
50 KiB
C
// SPDX-License-Identifier: GPL-2.0-only
|
|
/*
|
|
* FF-A v1.0 proxy to filter out invalid memory-sharing SMC calls issued by
|
|
* the host. FF-A is a slightly more palatable abbreviation of "Arm Firmware
|
|
* Framework for Arm A-profile", which is specified by Arm in document
|
|
* number DEN0077.
|
|
*
|
|
* Copyright (C) 2022 - Google LLC
|
|
* Author: Andrew Walbran <[email protected]>
|
|
*
|
|
* This driver hooks into the SMC trapping logic for the host and intercepts
|
|
* all calls falling within the FF-A range. Each call is either:
|
|
*
|
|
* - Forwarded on unmodified to the SPMD at EL3
|
|
* - Rejected as "unsupported"
|
|
* - Accompanied by a host stage-2 page-table check/update and reissued
|
|
*
|
|
* Consequently, any attempts by the host to make guest memory pages
|
|
* accessible to the secure world using FF-A will be detected either here
|
|
* (in the case that the memory is already owned by the guest) or during
|
|
* donation to the guest (in the case that the memory was previously shared
|
|
* with the secure world).
|
|
*
|
|
* To allow the rolling-back of page-table updates and FF-A calls in the
|
|
* event of failure, operations involving the RXTX buffers are locked for
|
|
* the duration and are therefore serialised.
|
|
*/
|
|
|
|
#include <linux/arm_ffa.h>
|
|
#include <asm/kvm_hypevents.h>
|
|
#include <asm/kvm_pkvm.h>
|
|
#include <kvm/arm_hypercalls.h>
|
|
#include <asm/virt.h>
|
|
|
|
#include <nvhe/arm-smccc.h>
|
|
#include <nvhe/alloc.h>
|
|
#include <nvhe/ffa.h>
|
|
#include <nvhe/mem_protect.h>
|
|
#include <nvhe/memory.h>
|
|
#include <nvhe/pkvm.h>
|
|
#include <nvhe/trap_handler.h>
|
|
#include <nvhe/spinlock.h>
|
|
|
|
#define VM_FFA_SUPPORTED(vcpu) ((vcpu)->kvm->arch.pkvm.ffa_support)
|
|
#define FFA_INVALID_SPM_HANDLE (BIT(63) - 1)
|
|
|
|
/* The maximum number of secure partitions that can register for VM availability */
|
|
#define FFA_MAX_VM_AVAIL_SPS (8)
|
|
#define FFA_VM_AVAIL_SPS_OOM (-2)
|
|
|
|
#define FFA_PART_VM_AVAIL_MASK (FFA_PARTITION_DIRECT_RECV |\
|
|
FFA_PARTITION_HYP_CREATE_VM |\
|
|
FFA_PARTITION_HYP_DESTROY_VM)
|
|
#define FFA_PART_SUPPORTS_VM_AVAIL (FFA_PART_VM_AVAIL_MASK)
|
|
|
|
/*
|
|
* A buffer to hold the maximum descriptor size we can see from the host,
|
|
* which is required when the SPMD returns a fragmented FFA_MEM_RETRIEVE_RESP
|
|
* when resolving the handle on the reclaim path.
|
|
*/
|
|
struct kvm_ffa_descriptor_buffer {
|
|
void *buf;
|
|
size_t len;
|
|
};
|
|
|
|
static struct kvm_ffa_descriptor_buffer ffa_desc_buf;
|
|
|
|
struct ffa_translation {
|
|
struct list_head node;
|
|
u64 ipa;
|
|
phys_addr_t pa;
|
|
};
|
|
|
|
struct ffa_handle {
|
|
u64 handle: 63;
|
|
u64 is_lend: 1;
|
|
};
|
|
|
|
/*
|
|
* Note that we don't currently lock these buffers explicitly, instead
|
|
* relying on the locking of the hyp FFA buffers.
|
|
*/
|
|
static struct kvm_ffa_buffers hyp_buffers;
|
|
static struct kvm_ffa_buffers host_buffers;
|
|
static u32 hyp_ffa_version;
|
|
static bool has_version_negotiated;
|
|
static bool has_hyp_ffa_buffer_mapped;
|
|
static bool has_host_signalled;
|
|
|
|
static struct ffa_handle *spm_handles, *spm_free_handle;
|
|
static u32 num_spm_handles;
|
|
|
|
static DEFINE_HYP_SPINLOCK(version_lock);
|
|
static DEFINE_HYP_SPINLOCK(kvm_ffa_hyp_lock);
|
|
|
|
/* Secure partitions that can receive VM availability messages */
|
|
struct kvm_ffa_vm_avail_sp {
|
|
u16 sp_id;
|
|
bool wants_create;
|
|
bool wants_destroy;
|
|
};
|
|
|
|
static struct kvm_ffa_vm_avail_sp vm_avail_sps[FFA_MAX_VM_AVAIL_SPS];
|
|
static int num_vm_avail_sps = -1;
|
|
|
|
static struct kvm_ffa_buffers *ffa_get_buffers(struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
if (!hyp_vcpu)
|
|
return &host_buffers;
|
|
|
|
return &pkvm_hyp_vcpu_to_hyp_vm(hyp_vcpu)->ffa_buf;
|
|
}
|
|
|
|
DECLARE_STATIC_KEY_FALSE(kvm_ffa_unmap_on_lend);
|
|
|
|
static struct ffa_handle *ffa_host_alloc_handle(void)
|
|
{
|
|
u32 i;
|
|
struct ffa_handle *handle;
|
|
|
|
if (spm_free_handle) {
|
|
WARN_ON(spm_free_handle < spm_handles ||
|
|
spm_free_handle >= (spm_handles + num_spm_handles));
|
|
handle = spm_free_handle;
|
|
spm_free_handle = NULL;
|
|
return handle;
|
|
}
|
|
|
|
for (i = 0; i < num_spm_handles; i++)
|
|
if (spm_handles[i].handle == FFA_INVALID_SPM_HANDLE)
|
|
break;
|
|
|
|
if (i == num_spm_handles)
|
|
return NULL;
|
|
|
|
return &spm_handles[i];
|
|
}
|
|
|
|
static struct ffa_handle *ffa_host_get_handle(u64 ffa_handle)
|
|
{
|
|
u32 i;
|
|
|
|
for (i = 0; i < num_spm_handles; i++)
|
|
if (spm_handles[i].handle == ffa_handle)
|
|
return &spm_handles[i];
|
|
return NULL;
|
|
}
|
|
|
|
static int ffa_host_clear_handle(u64 ffa_handle)
|
|
{
|
|
struct ffa_handle *entry = ffa_host_get_handle(ffa_handle);
|
|
|
|
if (!entry)
|
|
return -EINVAL;
|
|
|
|
entry->handle = FFA_INVALID_SPM_HANDLE;
|
|
spm_free_handle = entry;
|
|
return 0;
|
|
}
|
|
|
|
static void ffa_to_smccc_error(struct arm_smccc_1_2_regs *res, u64 ffa_errno)
|
|
{
|
|
*res = (struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_ERROR,
|
|
.a2 = ffa_errno,
|
|
};
|
|
}
|
|
|
|
static void ffa_to_smccc_res_prop(struct arm_smccc_1_2_regs *res, int ret, u64 prop)
|
|
{
|
|
if (ret == FFA_RET_SUCCESS) {
|
|
*res = (struct arm_smccc_1_2_regs) { .a0 = FFA_SUCCESS,
|
|
.a2 = prop };
|
|
} else {
|
|
ffa_to_smccc_error(res, ret);
|
|
}
|
|
}
|
|
|
|
static void ffa_to_smccc_res(struct arm_smccc_1_2_regs *res, int ret)
|
|
{
|
|
ffa_to_smccc_res_prop(res, ret, 0);
|
|
}
|
|
|
|
static void ffa_set_retval(struct kvm_cpu_context *ctxt,
|
|
struct arm_smccc_1_2_regs *res)
|
|
{
|
|
cpu_reg(ctxt, 0) = res->a0;
|
|
cpu_reg(ctxt, 1) = res->a1;
|
|
cpu_reg(ctxt, 2) = res->a2;
|
|
cpu_reg(ctxt, 3) = res->a3;
|
|
cpu_reg(ctxt, 4) = res->a4;
|
|
cpu_reg(ctxt, 5) = res->a5;
|
|
cpu_reg(ctxt, 6) = res->a6;
|
|
cpu_reg(ctxt, 7) = res->a7;
|
|
|
|
/*
|
|
* DEN0028C 2.6: SMC32/HVC32 call from aarch64 must preserve x8-x30.
|
|
*
|
|
* In FF-A 1.2, we cannot rely on the function ID sent by the caller to
|
|
* detect 32-bit calls because the CPU cycle management interfaces (e.g.
|
|
* FFA_MSG_WAIT, FFA_RUN) are 32-bit only but can have 64-bit responses.
|
|
*
|
|
* FFA-1.3 introduces 64-bit variants of the CPU cycle management
|
|
* interfaces. Moreover, FF-A 1.3 clarifies that SMC32 direct requests
|
|
* complete with SMC32 direct reponses which *should* allow us use the
|
|
* function ID sent by the caller to determine whether to return x8-x17.
|
|
*
|
|
* Note that we also cannot rely on function IDs in the response.
|
|
*
|
|
* Given the above, assume SMC64 and send back x0-x17 unconditionally
|
|
* as the passthrough code (__kvm_hyp_host_forward_smc) does the same.
|
|
*/
|
|
cpu_reg(ctxt, 8) = res->a8;
|
|
cpu_reg(ctxt, 9) = res->a9;
|
|
cpu_reg(ctxt, 10) = res->a10;
|
|
cpu_reg(ctxt, 11) = res->a11;
|
|
cpu_reg(ctxt, 12) = res->a12;
|
|
cpu_reg(ctxt, 13) = res->a13;
|
|
cpu_reg(ctxt, 14) = res->a14;
|
|
cpu_reg(ctxt, 15) = res->a15;
|
|
cpu_reg(ctxt, 16) = res->a16;
|
|
cpu_reg(ctxt, 17) = res->a17;
|
|
}
|
|
|
|
static int ffa_map_hyp_buffers(u64 ffa_page_count)
|
|
{
|
|
struct arm_smccc_1_2_regs res;
|
|
|
|
/*
|
|
* Ensure that the read of `has_hyp_ffa_buffer_mapped` is visible
|
|
* to other CPUs before proceeding.
|
|
*/
|
|
if (smp_load_acquire(&has_hyp_ffa_buffer_mapped))
|
|
return 0;
|
|
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_FN64_RXTX_MAP,
|
|
.a1 = hyp_virt_to_phys(hyp_buffers.tx),
|
|
.a2 = hyp_virt_to_phys(hyp_buffers.rx),
|
|
.a3 = ffa_page_count,
|
|
}, &res);
|
|
if (res.a0 != FFA_SUCCESS)
|
|
return res.a2;
|
|
|
|
/*
|
|
* Ensure that the write to `has_hyp_ffa_buffer_mapped` is visible
|
|
* to other CPUs after the previous operations.
|
|
*/
|
|
smp_store_release(&has_hyp_ffa_buffer_mapped, true);
|
|
|
|
return res.a0 == FFA_SUCCESS ? FFA_RET_SUCCESS : res.a2;
|
|
}
|
|
|
|
static int ffa_unmap_hyp_buffers(void)
|
|
{
|
|
struct arm_smccc_1_2_regs res;
|
|
|
|
/*
|
|
* Ensure that the read of `has_hyp_ffa_buffer_mapped` is visible
|
|
* to other CPUs before proceeding.
|
|
*/
|
|
if (!smp_load_acquire(&has_hyp_ffa_buffer_mapped))
|
|
return 0;
|
|
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_RXTX_UNMAP,
|
|
.a1 = HOST_FFA_ID,
|
|
}, &res);
|
|
if (res.a0 != FFA_SUCCESS)
|
|
return res.a2;
|
|
|
|
/*
|
|
* Ensure that the write to `has_hyp_ffa_buffer_mapped` is visible
|
|
* to other CPUs after the previous operations.
|
|
*/
|
|
smp_store_release(&has_hyp_ffa_buffer_mapped, false);
|
|
|
|
return FFA_RET_SUCCESS;
|
|
}
|
|
|
|
static void ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
|
|
u32 handle_hi, u32 fraglen, u32 endpoint_id)
|
|
{
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_MEM_FRAG_TX,
|
|
.a1 = handle_lo,
|
|
.a2 = handle_hi,
|
|
.a3 = fraglen,
|
|
.a4 = endpoint_id,
|
|
}, res);
|
|
}
|
|
|
|
static void ffa_mem_frag_rx(struct arm_smccc_1_2_regs *res, u32 handle_lo,
|
|
u32 handle_hi, u32 fragoff)
|
|
{
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_MEM_FRAG_RX,
|
|
.a1 = handle_lo,
|
|
.a2 = handle_hi,
|
|
.a3 = fragoff,
|
|
.a4 = HOST_FFA_ID,
|
|
}, res);
|
|
}
|
|
|
|
static void ffa_mem_xfer(struct arm_smccc_1_2_regs *res, u64 func_id, u32 len,
|
|
u32 fraglen)
|
|
{
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = func_id,
|
|
.a1 = len,
|
|
.a2 = fraglen,
|
|
}, res);
|
|
}
|
|
|
|
static void ffa_mem_reclaim(struct arm_smccc_1_2_regs *res, u32 handle_lo,
|
|
u32 handle_hi, u32 flags)
|
|
{
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_MEM_RECLAIM,
|
|
.a1 = handle_lo,
|
|
.a2 = handle_hi,
|
|
.a3 = flags,
|
|
}, res);
|
|
}
|
|
|
|
static void ffa_retrieve_req(struct arm_smccc_1_2_regs *res, u32 len)
|
|
{
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_FN64_MEM_RETRIEVE_REQ,
|
|
.a1 = len,
|
|
.a2 = len,
|
|
}, res);
|
|
}
|
|
|
|
static void ffa_rx_release(struct arm_smccc_1_2_regs *res)
|
|
{
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_RX_RELEASE,
|
|
}, res);
|
|
}
|
|
|
|
static int parse_vm_availability_resp(u32 partition_sz, u32 count)
|
|
{
|
|
struct ffa_partition_info *part;
|
|
u32 i, j, off;
|
|
bool supports_direct_recv, wants_create, wants_destroy;
|
|
|
|
if (num_vm_avail_sps >= 0)
|
|
return FFA_RET_SUCCESS;
|
|
if (num_vm_avail_sps == FFA_VM_AVAIL_SPS_OOM)
|
|
return FFA_RET_NO_MEMORY;
|
|
|
|
num_vm_avail_sps = 0;
|
|
for (i = 0; i < count; i++) {
|
|
if (check_mul_overflow(i, partition_sz, &off))
|
|
return FFA_RET_INVALID_PARAMETERS;
|
|
|
|
if (off >= KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)
|
|
return FFA_RET_INVALID_PARAMETERS;
|
|
|
|
part = hyp_buffers.rx + off;
|
|
supports_direct_recv = part->properties & FFA_PARTITION_DIRECT_RECV;
|
|
wants_create = part->properties & FFA_PARTITION_HYP_CREATE_VM;
|
|
wants_destroy = part->properties & FFA_PARTITION_HYP_DESTROY_VM;
|
|
|
|
if (supports_direct_recv && (wants_create || wants_destroy)) {
|
|
/* Check for duplicate SP IDs */
|
|
for (j = 0; j < num_vm_avail_sps; j++)
|
|
if (vm_avail_sps[j].sp_id == part->id)
|
|
break;
|
|
|
|
if (j == num_vm_avail_sps) {
|
|
if (num_vm_avail_sps >= FFA_MAX_VM_AVAIL_SPS) {
|
|
/* We ran out of space in the array */
|
|
num_vm_avail_sps = FFA_VM_AVAIL_SPS_OOM;
|
|
return FFA_RET_NO_MEMORY;
|
|
}
|
|
|
|
vm_avail_sps[num_vm_avail_sps].sp_id = part->id;
|
|
vm_avail_sps[num_vm_avail_sps].wants_create = wants_create;
|
|
vm_avail_sps[num_vm_avail_sps].wants_destroy = wants_destroy;
|
|
num_vm_avail_sps++;
|
|
}
|
|
}
|
|
}
|
|
|
|
return FFA_RET_SUCCESS;
|
|
}
|
|
|
|
static int kvm_notify_vm_availability(uint16_t vm_handle, struct kvm_ffa_buffers *ffa_buf,
|
|
u32 availability_msg)
|
|
{
|
|
int i;
|
|
struct arm_smccc_1_2_regs res;
|
|
u64 avail_bit = availability_msg != FFA_VM_DESTRUCTION_MSG;
|
|
|
|
for (i = 0; i < num_vm_avail_sps; i++) {
|
|
u64 sp_mask = 1UL << i;
|
|
u64 avail_value = avail_bit << i;
|
|
uint32_t dest = ((uint32_t)vm_avail_sps[i].sp_id << 16) | hyp_smp_processor_id();
|
|
|
|
if ((ffa_buf->vm_avail_bitmap & sp_mask) == avail_value &&
|
|
!(ffa_buf->vm_creating_bitmap & sp_mask))
|
|
continue;
|
|
|
|
if (avail_bit && !vm_avail_sps[i].wants_create) {
|
|
/*
|
|
* The SP did not ask for creation messages,
|
|
* so just mark this VM as available and
|
|
* continue
|
|
*/
|
|
ffa_buf->vm_avail_bitmap |= avail_value;
|
|
continue;
|
|
} else if (!avail_bit && !vm_avail_sps[i].wants_destroy) {
|
|
/*
|
|
* The SP did not ask for destruction messages,
|
|
* so just mark this VM as not available and
|
|
* continue
|
|
*/
|
|
ffa_buf->vm_avail_bitmap &= ~sp_mask;
|
|
continue;
|
|
}
|
|
|
|
/*
|
|
* Give the SP some cycles in advance,
|
|
* in case it got interrupted the last time.
|
|
*
|
|
* Some TEEs return NOT_SUPPORTED instead.
|
|
* If that happens, ignore the error and continue.
|
|
*/
|
|
res = (struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_RUN,
|
|
.a1 = dest,
|
|
};
|
|
nvhe_arm_smccc_1_2_smc(&res, &res);
|
|
if (res.a0 == FFA_ERROR && (int)res.a2 != FFA_RET_NOT_SUPPORTED)
|
|
return ffa_to_linux_errno(res.a2);
|
|
else if (res.a0 == FFA_INTERRUPT)
|
|
return -EINTR;
|
|
|
|
if (availability_msg == FFA_VM_DESTRUCTION_MSG &&
|
|
(ffa_buf->vm_creating_bitmap & sp_mask)) {
|
|
/*
|
|
* If we sent the initial creation message for this VM
|
|
* but never got the success response from the TEE, we
|
|
* need to keep trying to create it until it works.
|
|
* Otherwise we cannot destroy it.
|
|
*
|
|
* TODO: this is not triggered for SPs that requested only
|
|
* creation messages (but not destruction). In that case,
|
|
* we will never retry the creation message, and the SP
|
|
* will probably leak its state for the pending VM.
|
|
*/
|
|
res = (struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_MSG_SEND_DIRECT_REQ,
|
|
.a1 = vm_avail_sps[i].sp_id,
|
|
.a2 = FFA_VM_CREATION_MSG,
|
|
.a3 = HANDLE_LOW(FFA_INVALID_HANDLE),
|
|
.a4 = HANDLE_HIGH(FFA_INVALID_HANDLE),
|
|
.a5 = vm_handle,
|
|
};
|
|
nvhe_arm_smccc_1_2_smc(&res, &res);
|
|
|
|
if (res.a0 != FFA_MSG_SEND_DIRECT_RESP)
|
|
return -EINVAL;
|
|
if (res.a3 != FFA_RET_SUCCESS)
|
|
return ffa_to_linux_errno(res.a3);
|
|
|
|
/* Creation completed successfully, clear the flag */
|
|
ffa_buf->vm_creating_bitmap &= ~sp_mask;
|
|
}
|
|
|
|
res = (struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_MSG_SEND_DIRECT_REQ,
|
|
.a1 = vm_avail_sps[i].sp_id,
|
|
.a2 = availability_msg,
|
|
.a3 = HANDLE_LOW(FFA_INVALID_HANDLE),
|
|
.a4 = HANDLE_HIGH(FFA_INVALID_HANDLE),
|
|
.a5 = vm_handle,
|
|
};
|
|
nvhe_arm_smccc_1_2_smc(&res, &res);
|
|
if (res.a0 != FFA_MSG_SEND_DIRECT_RESP)
|
|
return -EINVAL;
|
|
|
|
switch ((int)res.a3) {
|
|
case FFA_RET_SUCCESS:
|
|
ffa_buf->vm_avail_bitmap &= ~sp_mask;
|
|
ffa_buf->vm_avail_bitmap |= avail_value;
|
|
ffa_buf->vm_creating_bitmap &= ~sp_mask;
|
|
break;
|
|
|
|
case FFA_RET_INTERRUPTED:
|
|
case FFA_RET_RETRY:
|
|
if (availability_msg == FFA_VM_CREATION_MSG)
|
|
ffa_buf->vm_creating_bitmap |= sp_mask;
|
|
|
|
fallthrough;
|
|
default:
|
|
return ffa_to_linux_errno(res.a3);
|
|
}
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void do_ffa_rxtx_map(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(phys_addr_t, tx, ctxt, 1);
|
|
DECLARE_REG(phys_addr_t, rx, ctxt, 2);
|
|
DECLARE_REG(u32, npages, ctxt, 3);
|
|
int ret = 0;
|
|
void *rx_virt, *tx_virt;
|
|
struct kvm_ffa_buffers *ffa_buf;
|
|
|
|
if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out;
|
|
}
|
|
|
|
if (!PAGE_ALIGNED(tx) || !PAGE_ALIGNED(rx)) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out;
|
|
}
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_buf = ffa_get_buffers(hyp_vcpu);
|
|
if (ffa_buf->tx) {
|
|
ret = FFA_RET_DENIED;
|
|
goto out_unlock;
|
|
}
|
|
|
|
/*
|
|
* Map our hypervisor buffers into the SPMD before mapping and
|
|
* pinning the host buffers in our own address space.
|
|
*/
|
|
ret = ffa_map_hyp_buffers(npages);
|
|
if (ret)
|
|
goto out_unlock;
|
|
|
|
ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(tx));
|
|
if (ret) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto err_unmap;
|
|
}
|
|
|
|
ret = __pkvm_host_share_hyp(hyp_phys_to_pfn(rx));
|
|
if (ret) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto err_unshare_tx;
|
|
}
|
|
|
|
tx_virt = hyp_phys_to_virt(tx);
|
|
ret = hyp_pin_shared_mem(tx_virt, tx_virt + 1);
|
|
if (ret) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto err_unshare_rx;
|
|
}
|
|
|
|
rx_virt = hyp_phys_to_virt(rx);
|
|
ret = hyp_pin_shared_mem(rx_virt, rx_virt + 1);
|
|
if (ret) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto err_unpin_tx;
|
|
}
|
|
|
|
ffa_buf->tx = tx_virt;
|
|
ffa_buf->rx = rx_virt;
|
|
|
|
out_unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
out:
|
|
ffa_to_smccc_res(res, ret);
|
|
return;
|
|
|
|
err_unpin_tx:
|
|
hyp_unpin_shared_mem(tx_virt, tx_virt + 1);
|
|
err_unshare_rx:
|
|
__pkvm_host_unshare_hyp(hyp_phys_to_pfn(rx));
|
|
err_unshare_tx:
|
|
__pkvm_host_unshare_hyp(hyp_phys_to_pfn(tx));
|
|
err_unmap:
|
|
ffa_unmap_hyp_buffers();
|
|
goto out_unlock;
|
|
}
|
|
|
|
static int __handle_mem_protect_err(struct pkvm_hyp_vcpu *hyp_vcpu, u64 ipa, int ret)
|
|
{
|
|
struct kvm_hyp_req *req;
|
|
|
|
switch (ret) {
|
|
case -EFAULT:
|
|
req = pkvm_hyp_req_reserve(hyp_vcpu, KVM_HYP_REQ_TYPE_MAP);
|
|
if (!req)
|
|
return -ENOSPC;
|
|
|
|
req->map.guest_ipa = ipa;
|
|
req->map.size = PAGE_SIZE;
|
|
break;
|
|
case -E2BIG:
|
|
req = pkvm_hyp_req_reserve(hyp_vcpu, KVM_HYP_REQ_TYPE_SPLIT);
|
|
if (!req)
|
|
return -ENOSPC;
|
|
|
|
req->split.guest_ipa = ALIGN_DOWN(ipa, PMD_SIZE);
|
|
req->split.size = PMD_SIZE;
|
|
break;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int do_ffa_rxtx_guest_map(struct kvm_cpu_context *ctxt, struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(phys_addr_t, tx, ctxt, 1);
|
|
DECLARE_REG(phys_addr_t, rx, ctxt, 2);
|
|
DECLARE_REG(u32, npages, ctxt, 3);
|
|
int ret = 0;
|
|
u64 rx_va, tx_va;
|
|
struct kvm_ffa_buffers *ffa_buf;
|
|
|
|
if (npages != (KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE)
|
|
return -EINVAL;
|
|
|
|
if (!PAGE_ALIGNED(tx) || !PAGE_ALIGNED(rx))
|
|
return -EINVAL;
|
|
|
|
ret = __pkvm_guest_share_hyp_page(hyp_vcpu, tx, &tx_va);
|
|
if (ret)
|
|
goto out_err;
|
|
|
|
ret = __pkvm_guest_share_hyp_page(hyp_vcpu, rx, &rx_va);
|
|
if (ret)
|
|
goto out_err_with_tx;
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_buf = ffa_get_buffers(hyp_vcpu);
|
|
if (ffa_buf->tx) {
|
|
ret = -EACCES;
|
|
goto out_unlock;
|
|
}
|
|
|
|
ffa_buf->tx = (void *)tx_va;
|
|
ffa_buf->rx = (void *)rx_va;
|
|
ffa_buf->tx_ipa = tx;
|
|
ffa_buf->rx_ipa = rx;
|
|
out_unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
return ret;
|
|
out_err_with_tx:
|
|
WARN_ON(__pkvm_guest_unshare_hyp_page(hyp_vcpu, tx));
|
|
out_err:
|
|
ret = __handle_mem_protect_err(hyp_vcpu, tx, ret);
|
|
ret = __handle_mem_protect_err(hyp_vcpu, rx, ret);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void do_ffa_rxtx_unmap(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(u32, id, ctxt, 1);
|
|
int ret = 0;
|
|
struct kvm_ffa_buffers *ffa_buf;
|
|
|
|
if (hyp_vcpu_to_ffa_handle(hyp_vcpu) != id) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out;
|
|
}
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_buf = ffa_get_buffers(hyp_vcpu);
|
|
if (!ffa_buf->tx) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out_unlock;
|
|
}
|
|
|
|
if (!hyp_vcpu) {
|
|
hyp_unpin_shared_mem(ffa_buf->tx, ffa_buf->tx + 1);
|
|
WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(ffa_buf->tx)));
|
|
|
|
hyp_unpin_shared_mem(ffa_buf->rx, ffa_buf->rx + 1);
|
|
WARN_ON(__pkvm_host_unshare_hyp(hyp_virt_to_pfn(ffa_buf->rx)));
|
|
|
|
ffa_unmap_hyp_buffers();
|
|
} else {
|
|
WARN_ON(__pkvm_guest_unshare_hyp_page(hyp_vcpu, ffa_buf->tx_ipa));
|
|
WARN_ON(__pkvm_guest_unshare_hyp_page(hyp_vcpu, ffa_buf->rx_ipa));
|
|
}
|
|
|
|
ffa_buf->rx = NULL;
|
|
ffa_buf->tx = NULL;
|
|
|
|
out_unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
out:
|
|
ffa_to_smccc_res(res, ret);
|
|
}
|
|
|
|
static u32 __ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges,
|
|
u32 nranges, bool is_lend)
|
|
{
|
|
u32 i;
|
|
int ret;
|
|
|
|
for (i = 0; i < nranges; ++i) {
|
|
struct ffa_mem_region_addr_range *range = &ranges[i];
|
|
u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
|
|
u64 pfn = hyp_phys_to_pfn(range->address);
|
|
|
|
if (!PAGE_ALIGNED(sz))
|
|
break;
|
|
|
|
if (static_branch_unlikely(&kvm_ffa_unmap_on_lend) && is_lend)
|
|
ret = __pkvm_host_donate_ffa(pfn, sz / PAGE_SIZE);
|
|
else
|
|
ret = __pkvm_host_share_ffa(pfn, sz / PAGE_SIZE);
|
|
if (ret)
|
|
break;
|
|
}
|
|
|
|
return i;
|
|
}
|
|
|
|
/*
|
|
* Verify if the page is lent on shared and unshare it with FF-A.
|
|
* On success, return the number of *unshared* pages and store in the
|
|
* is_lend argument whether the range was shared or lent.
|
|
*/
|
|
static u32 __ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges,
|
|
u32 nranges, bool is_lend)
|
|
{
|
|
u32 i;
|
|
int ret;
|
|
|
|
for (i = 0; i < nranges; ++i) {
|
|
struct ffa_mem_region_addr_range *range = &ranges[i];
|
|
u64 sz = (u64)range->pg_cnt * FFA_PAGE_SIZE;
|
|
u64 pfn = hyp_phys_to_pfn(range->address);
|
|
|
|
if (!PAGE_ALIGNED(sz))
|
|
break;
|
|
|
|
if (static_branch_unlikely(&kvm_ffa_unmap_on_lend) && is_lend)
|
|
ret = __pkvm_host_reclaim_ffa(pfn, sz / PAGE_SIZE);
|
|
else
|
|
|
|
ret = __pkvm_host_unshare_ffa(pfn, sz / PAGE_SIZE);
|
|
if (ret)
|
|
break;
|
|
}
|
|
|
|
return i;
|
|
}
|
|
|
|
static int ffa_store_translation(struct ffa_mem_transfer *transfer, u64 ipa, phys_addr_t pa)
|
|
{
|
|
struct ffa_translation *tr;
|
|
|
|
tr = hyp_alloc(sizeof(struct ffa_translation));
|
|
if (!tr)
|
|
return -ENOMEM;
|
|
|
|
tr->ipa = ipa;
|
|
tr->pa = pa;
|
|
list_add(&tr->node, &transfer->translations);
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void ffa_guest_unshare_ranges(struct pkvm_hyp_vcpu *vcpu,
|
|
struct ffa_mem_transfer *transfer)
|
|
{
|
|
struct ffa_translation *translation, *tmp;
|
|
|
|
list_for_each_entry_safe(translation, tmp, &transfer->translations, node) {
|
|
WARN_ON(__pkvm_guest_unshare_ffa_page(vcpu, translation->ipa));
|
|
list_del(&translation->node);
|
|
hyp_free(translation);
|
|
}
|
|
}
|
|
|
|
static int ffa_guest_share_ranges(struct ffa_mem_region_addr_range *ranges,
|
|
u32 nranges, struct pkvm_hyp_vcpu *vcpu,
|
|
struct ffa_composite_mem_region *out_region,
|
|
size_t reg_len,
|
|
struct ffa_mem_transfer *transfer)
|
|
{
|
|
struct ffa_mem_region_addr_range *range;
|
|
struct ffa_mem_region_addr_range *buf = out_region->constituents;
|
|
int i, j, ret;
|
|
u32 mem_region_idx = 0;
|
|
u64 ipa, pa, offset;
|
|
|
|
for (i = 0; i < nranges; i++) {
|
|
range = &ranges[i];
|
|
for (j = 0; j < range->pg_cnt; j++) {
|
|
if (mem_region_idx * sizeof(struct ffa_mem_region_addr_range) >= reg_len) {
|
|
ret = -EINVAL;
|
|
goto unshare;
|
|
}
|
|
|
|
if (check_mul_overflow(j, PAGE_SIZE, &offset) ||
|
|
check_add_overflow(range->address, offset, &ipa)) {
|
|
ret = -EINVAL;
|
|
goto unshare;
|
|
}
|
|
|
|
ret = __handle_mem_protect_err(vcpu, ipa,
|
|
__pkvm_guest_share_ffa_page(vcpu, ipa, &pa));
|
|
if (ret)
|
|
goto unshare;
|
|
|
|
ret = ffa_store_translation(transfer, ipa, pa);
|
|
if (ret) {
|
|
WARN_ON(__pkvm_guest_unshare_ffa_page(vcpu, ipa));
|
|
goto unshare;
|
|
}
|
|
|
|
buf[mem_region_idx].address = pa;
|
|
buf[mem_region_idx].pg_cnt = 1;
|
|
|
|
if (mem_region_idx + 1 < mem_region_idx) {
|
|
ret = -EINVAL;
|
|
goto unshare;
|
|
}
|
|
|
|
mem_region_idx++;
|
|
}
|
|
}
|
|
|
|
out_region->addr_range_cnt = mem_region_idx;
|
|
return 0;
|
|
unshare:
|
|
ffa_guest_unshare_ranges(vcpu, transfer);
|
|
return ret;
|
|
}
|
|
|
|
static int ffa_host_share_ranges(struct ffa_mem_region_addr_range *ranges,
|
|
u32 nranges, bool is_lend)
|
|
{
|
|
u32 nshared = __ffa_host_share_ranges(ranges, nranges, is_lend);
|
|
int ret = 0;
|
|
|
|
if (nshared != nranges) {
|
|
WARN_ON(__ffa_host_unshare_ranges(ranges, nshared, is_lend) != nshared);
|
|
ret = -EACCES;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int ffa_host_unshare_ranges(struct ffa_mem_region_addr_range *ranges,
|
|
u32 nranges, bool is_lend)
|
|
{
|
|
int ret = 0;
|
|
u32 nunshared = __ffa_host_unshare_ranges(ranges, nranges, is_lend);
|
|
|
|
if (nunshared != nranges) {
|
|
WARN_ON(__ffa_host_share_ranges(ranges, nunshared, is_lend) != nunshared);
|
|
ret = -EACCES;
|
|
}
|
|
|
|
return ret;
|
|
}
|
|
|
|
static void do_ffa_mem_frag_tx(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(u32, handle_lo, ctxt, 1);
|
|
DECLARE_REG(u32, handle_hi, ctxt, 2);
|
|
DECLARE_REG(u32, fraglen, ctxt, 3);
|
|
DECLARE_REG(u32, endpoint_id, ctxt, 4);
|
|
struct ffa_mem_region_addr_range *buf;
|
|
int ret = FFA_RET_INVALID_PARAMETERS;
|
|
u32 nr_ranges;
|
|
struct kvm_ffa_buffers *ffa_buf;
|
|
bool is_lend = false;
|
|
u64 host_handle = PACK_HANDLE(handle_lo, handle_hi);
|
|
struct ffa_handle *entry;
|
|
|
|
if (fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)
|
|
goto out;
|
|
|
|
if (fraglen % sizeof(*buf))
|
|
goto out;
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_buf = ffa_get_buffers(hyp_vcpu);
|
|
if (!ffa_buf->tx)
|
|
goto out_unlock;
|
|
|
|
buf = hyp_buffers.tx;
|
|
memcpy(buf, ffa_buf->tx, fraglen);
|
|
nr_ranges = fraglen / sizeof(*buf);
|
|
|
|
if (static_branch_unlikely(&kvm_ffa_unmap_on_lend)) {
|
|
entry = ffa_host_get_handle(host_handle);
|
|
if (!entry) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
is_lend = entry->is_lend;
|
|
}
|
|
|
|
ret = ffa_host_share_ranges(buf, nr_ranges, is_lend);
|
|
if (ret) {
|
|
struct ffa_mem_transfer *transfer = __pkvm_get_vm_ffa_transfer(host_handle);
|
|
WARN_ON(transfer);
|
|
|
|
/*
|
|
* We're effectively aborting the transaction, so we need
|
|
* to restore the global state back to what it was prior to
|
|
* transmission of the first fragment.
|
|
*/
|
|
ffa_mem_reclaim(res, handle_lo, handle_hi, 0);
|
|
WARN_ON(res->a0 != FFA_SUCCESS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
ffa_mem_frag_tx(res, handle_lo, handle_hi, fraglen, endpoint_id);
|
|
if (res->a0 != FFA_SUCCESS && res->a0 != FFA_MEM_FRAG_RX)
|
|
WARN_ON(ffa_host_unshare_ranges(buf, nr_ranges, is_lend));
|
|
|
|
out_unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
out:
|
|
if (ret)
|
|
ffa_to_smccc_res(res, ret);
|
|
|
|
/*
|
|
* If for any reason this did not succeed, we're in trouble as we have
|
|
* now lost the content of the previous fragments and we can't rollback
|
|
* the host stage-2 changes. The pages previously marked as shared will
|
|
* remain stuck in that state forever, hence preventing the host from
|
|
* sharing/donating them again and may possibly lead to subsequent
|
|
* failures, but this will not compromise confidentiality.
|
|
*/
|
|
return;
|
|
}
|
|
|
|
static bool is_page_count_valid(struct ffa_composite_mem_region *reg,
|
|
u32 nranges)
|
|
{
|
|
u32 i, pg_cnt = 0, new_pg_cnt;
|
|
|
|
for (i = 0; i < nranges; i++) {
|
|
new_pg_cnt = pg_cnt + reg->constituents[i].pg_cnt;
|
|
if (new_pg_cnt < pg_cnt)
|
|
return false;
|
|
|
|
pg_cnt = new_pg_cnt;
|
|
}
|
|
|
|
return pg_cnt == reg->total_pg_cnt;
|
|
}
|
|
|
|
static int __do_ffa_mem_xfer(const u64 func_id,
|
|
struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(u32, len, ctxt, 1);
|
|
DECLARE_REG(u32, fraglen, ctxt, 2);
|
|
DECLARE_REG(u64, addr_mbz, ctxt, 3);
|
|
DECLARE_REG(u32, npages_mbz, ctxt, 4);
|
|
struct ffa_mem_region_attributes *ep_mem_access;
|
|
struct ffa_composite_mem_region *reg, *temp_reg;
|
|
struct ffa_mem_region *buf;
|
|
struct kvm_ffa_buffers *ffa_buf;
|
|
u32 offset, nr_ranges, checked_offset;
|
|
int ret = 0;
|
|
struct ffa_mem_transfer *transfer = NULL;
|
|
u64 ffa_handle;
|
|
bool is_lend = func_id == FFA_FN64_MEM_LEND;
|
|
struct ffa_handle *handle = NULL;
|
|
|
|
if (addr_mbz || npages_mbz || fraglen > len ||
|
|
fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
return 0;
|
|
}
|
|
|
|
if (fraglen < sizeof(struct ffa_mem_region) +
|
|
sizeof(struct ffa_mem_region_attributes)) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
return 0;
|
|
}
|
|
|
|
if (hyp_vcpu) {
|
|
/* Reject the fragmentation API for the guest */
|
|
if (len != fraglen) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
return 0;
|
|
}
|
|
|
|
transfer = hyp_alloc(sizeof(struct ffa_mem_transfer));
|
|
if (!transfer)
|
|
return -ENOMEM;
|
|
|
|
INIT_LIST_HEAD(&transfer->translations);
|
|
}
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_buf = ffa_get_buffers(hyp_vcpu);
|
|
if (!ffa_buf->tx) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
if (len > ffa_desc_buf.len) {
|
|
ffa_to_smccc_error(res, FFA_RET_NO_MEMORY);
|
|
goto out_unlock;
|
|
}
|
|
|
|
buf = hyp_buffers.tx;
|
|
memcpy(buf, ffa_buf->tx, fraglen);
|
|
|
|
ep_mem_access = (void *)buf +
|
|
ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
|
|
offset = ep_mem_access->composite_off;
|
|
if (!offset || buf->ep_count != 1 || buf->sender_id != hyp_vcpu_to_ffa_handle(hyp_vcpu)) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
if (check_add_overflow(offset, sizeof(struct ffa_composite_mem_region), &checked_offset)) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out_unlock;
|
|
}
|
|
|
|
if (fraglen < checked_offset) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
reg = (void *)buf + offset;
|
|
nr_ranges = ((void *)buf + fraglen) - (void *)reg->constituents;
|
|
if (nr_ranges % sizeof(reg->constituents[0])) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
nr_ranges /= sizeof(reg->constituents[0]);
|
|
if (hyp_vcpu) {
|
|
if (!is_page_count_valid(reg, nr_ranges)) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
size_t translated_sz = reg->total_pg_cnt * sizeof(struct ffa_mem_region_addr_range)
|
|
+ offset;
|
|
if (translated_sz > PAGE_SIZE) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_unlock;
|
|
}
|
|
|
|
memcpy(ffa_desc_buf.buf, buf, offset);
|
|
temp_reg = ffa_desc_buf.buf + offset;
|
|
ret = ffa_guest_share_ranges(reg->constituents, nr_ranges, hyp_vcpu,
|
|
temp_reg, ffa_desc_buf.len - offset,
|
|
transfer);
|
|
if (!ret) {
|
|
/* Re-adjust the size of the transfer after painting with PAs */
|
|
if (temp_reg->addr_range_cnt > reg->addr_range_cnt) {
|
|
u32 extra_sz = (temp_reg->addr_range_cnt - reg->addr_range_cnt) *
|
|
sizeof(struct ffa_mem_region_addr_range);
|
|
fraglen += extra_sz;
|
|
len += extra_sz;
|
|
|
|
nr_ranges = reg->addr_range_cnt = temp_reg->addr_range_cnt;
|
|
}
|
|
|
|
memcpy(reg->constituents, temp_reg->constituents,
|
|
temp_reg->addr_range_cnt * sizeof(struct ffa_mem_region_addr_range));
|
|
}
|
|
} else
|
|
ret = ffa_host_share_ranges(reg->constituents, nr_ranges, is_lend);
|
|
if (ret)
|
|
goto out_unlock;
|
|
|
|
if (!hyp_vcpu && static_branch_unlikely(&kvm_ffa_unmap_on_lend)) {
|
|
handle = ffa_host_alloc_handle();
|
|
if (!handle) {
|
|
ret = -ENOSPC;
|
|
goto out_unlock;
|
|
}
|
|
}
|
|
|
|
ffa_mem_xfer(res, func_id, len, fraglen);
|
|
if (fraglen != len) {
|
|
if (res->a0 != FFA_MEM_FRAG_RX)
|
|
goto err_unshare;
|
|
|
|
if (res->a3 != fraglen)
|
|
goto err_unshare;
|
|
|
|
ffa_handle = PACK_HANDLE(res->a1, res->a2);
|
|
} else if (res->a0 == FFA_SUCCESS) {
|
|
ffa_handle = PACK_HANDLE(res->a2, res->a3);
|
|
} else {
|
|
goto err_unshare;
|
|
}
|
|
|
|
if (hyp_vcpu && transfer) {
|
|
transfer->ffa_handle = ffa_handle;
|
|
list_add(&transfer->node, &ffa_buf->xfer_list);
|
|
} else if (handle) {
|
|
handle->handle = ffa_handle;
|
|
handle->is_lend = is_lend;
|
|
}
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
return 0;
|
|
out_unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
if (transfer)
|
|
hyp_free(transfer);
|
|
if (ret)
|
|
ffa_to_smccc_res(res, linux_errno_to_ffa(ret));
|
|
return ret;
|
|
err_unshare:
|
|
if (hyp_vcpu)
|
|
ffa_guest_unshare_ranges(hyp_vcpu, transfer);
|
|
else
|
|
WARN_ON(ffa_host_unshare_ranges(reg->constituents, nr_ranges, is_lend));
|
|
goto out_unlock;
|
|
}
|
|
|
|
#define do_ffa_mem_xfer(fid, res, ctxt, hyp_vcpu) ({\
|
|
BUILD_BUG_ON((fid) != FFA_FN64_MEM_SHARE && \
|
|
(fid) != FFA_FN64_MEM_LEND); \
|
|
__do_ffa_mem_xfer((fid), (res), (ctxt), (hyp_vcpu));\
|
|
})
|
|
|
|
struct ffa_mem_transfer *find_transfer_by_handle(u64 ffa_handle, struct kvm_ffa_buffers *buf)
|
|
{
|
|
struct ffa_mem_transfer *transfer;
|
|
|
|
list_for_each_entry(transfer, &buf->xfer_list, node)
|
|
if (transfer->ffa_handle == ffa_handle)
|
|
return transfer;
|
|
return NULL;
|
|
}
|
|
|
|
static void do_ffa_mem_reclaim(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(u32, handle_lo, ctxt, 1);
|
|
DECLARE_REG(u32, handle_hi, ctxt, 2);
|
|
DECLARE_REG(u32, flags, ctxt, 3);
|
|
struct ffa_mem_region_attributes *ep_mem_access;
|
|
struct ffa_composite_mem_region *reg;
|
|
u32 offset, len, fraglen, fragoff;
|
|
struct ffa_mem_region *buf;
|
|
int ret = 0;
|
|
u64 handle;
|
|
struct ffa_mem_transfer *transfer = NULL;
|
|
struct kvm_ffa_buffers *ffa_buf;
|
|
struct ffa_handle *entry;
|
|
bool is_lend = false;
|
|
|
|
handle = PACK_HANDLE(handle_lo, handle_hi);
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_buf = ffa_get_buffers(hyp_vcpu);
|
|
if (hyp_vcpu) {
|
|
transfer = find_transfer_by_handle(handle, ffa_buf);
|
|
if (!transfer) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out_unlock;
|
|
}
|
|
|
|
goto out_reclaim;
|
|
} else {
|
|
transfer = __pkvm_get_vm_ffa_transfer(handle);
|
|
|
|
/* Prevent the host from replicating a transfer handle used by the guest */
|
|
WARN_ON(transfer);
|
|
|
|
if (static_branch_unlikely(&kvm_ffa_unmap_on_lend)) {
|
|
entry = ffa_host_get_handle(handle);
|
|
if (!entry) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out_unlock;
|
|
}
|
|
|
|
is_lend = entry->is_lend;
|
|
}
|
|
}
|
|
|
|
buf = hyp_buffers.tx;
|
|
*buf = (struct ffa_mem_region) {
|
|
.handle = handle,
|
|
};
|
|
|
|
ffa_retrieve_req(res, sizeof(*buf));
|
|
buf = hyp_buffers.rx;
|
|
if (res->a0 != FFA_MEM_RETRIEVE_RESP)
|
|
goto out_unlock;
|
|
|
|
len = res->a1;
|
|
fraglen = res->a2;
|
|
|
|
ep_mem_access = (void *)buf +
|
|
ffa_mem_desc_offset(buf, 0, hyp_ffa_version);
|
|
offset = ep_mem_access->composite_off;
|
|
/*
|
|
* We can trust the SPMD to get this right, but let's at least
|
|
* check that we end up with something that doesn't look _completely_
|
|
* bogus.
|
|
*/
|
|
if (WARN_ON(offset > len ||
|
|
fraglen > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE)) {
|
|
ret = FFA_RET_ABORTED;
|
|
ffa_rx_release(res);
|
|
goto out_unlock;
|
|
}
|
|
|
|
if (len > ffa_desc_buf.len) {
|
|
ret = FFA_RET_NO_MEMORY;
|
|
ffa_rx_release(res);
|
|
goto out_unlock;
|
|
}
|
|
|
|
buf = ffa_desc_buf.buf;
|
|
memcpy(buf, hyp_buffers.rx, fraglen);
|
|
ffa_rx_release(res);
|
|
|
|
for (fragoff = fraglen; fragoff < len; fragoff += fraglen) {
|
|
ffa_mem_frag_rx(res, handle_lo, handle_hi, fragoff);
|
|
if (res->a0 != FFA_MEM_FRAG_TX) {
|
|
ret = FFA_RET_INVALID_PARAMETERS;
|
|
goto out_unlock;
|
|
}
|
|
|
|
fraglen = res->a3;
|
|
memcpy((void *)buf + fragoff, hyp_buffers.rx, fraglen);
|
|
ffa_rx_release(res);
|
|
}
|
|
|
|
out_reclaim:
|
|
ffa_mem_reclaim(res, handle_lo, handle_hi, flags);
|
|
if (res->a0 != FFA_SUCCESS)
|
|
goto out_unlock;
|
|
|
|
/* If the SPMD was happy, then we should be too. */
|
|
if (hyp_vcpu)
|
|
ffa_guest_unshare_ranges(hyp_vcpu, transfer);
|
|
else {
|
|
reg = (void *)buf + offset;
|
|
WARN_ON(ffa_host_unshare_ranges(reg->constituents,
|
|
reg->addr_range_cnt, is_lend));
|
|
if (static_branch_unlikely(&kvm_ffa_unmap_on_lend))
|
|
ffa_host_clear_handle(handle);
|
|
}
|
|
|
|
if (transfer) {
|
|
list_del(&transfer->node);
|
|
hyp_free(transfer);
|
|
}
|
|
out_unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
|
|
if (ret)
|
|
ffa_to_smccc_res(res, ret);
|
|
}
|
|
|
|
/*
|
|
* Is a given FFA function supported, either by forwarding on directly
|
|
* or by handling at EL2?
|
|
*/
|
|
static bool ffa_call_supported(u64 func_id)
|
|
{
|
|
switch (func_id) {
|
|
/* Unsupported memory management calls */
|
|
case FFA_FN64_MEM_RETRIEVE_REQ:
|
|
case FFA_MEM_RETRIEVE_RESP:
|
|
case FFA_MEM_RELINQUISH:
|
|
case FFA_MEM_OP_PAUSE:
|
|
case FFA_MEM_OP_RESUME:
|
|
case FFA_MEM_FRAG_RX:
|
|
case FFA_FN64_MEM_DONATE:
|
|
/* Indirect message passing via RX/TX buffers */
|
|
case FFA_MSG_SEND:
|
|
case FFA_MSG_POLL:
|
|
case FFA_MSG_WAIT:
|
|
/* 32-bit variants of 64-bit calls */
|
|
case FFA_MSG_SEND_DIRECT_RESP:
|
|
case FFA_RXTX_MAP:
|
|
case FFA_MEM_DONATE:
|
|
case FFA_MEM_RETRIEVE_REQ:
|
|
/* Optional notification interfaces added in FF-A 1.1 */
|
|
case FFA_NOTIFICATION_BITMAP_CREATE:
|
|
case FFA_NOTIFICATION_BITMAP_DESTROY:
|
|
case FFA_NOTIFICATION_BIND:
|
|
case FFA_NOTIFICATION_UNBIND:
|
|
case FFA_NOTIFICATION_SET:
|
|
case FFA_NOTIFICATION_GET:
|
|
case FFA_NOTIFICATION_INFO_GET:
|
|
return false;
|
|
/* Optional interfaces added in FF-A 1.2 */
|
|
case FFA_MSG_SEND_DIRECT_REQ2: /* Optional per 7.5.1 */
|
|
return hyp_ffa_version >= FFA_VERSION_1_2;
|
|
case FFA_MSG_SEND_DIRECT_RESP2: /* Optional per 7.5.1 */
|
|
case FFA_CONSOLE_LOG: /* Optional per 13.1: not in Table 13.1 */
|
|
case FFA_PARTITION_INFO_GET_REGS: /* Optional for virtual instances per 13.1 */
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
static bool do_ffa_features(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt)
|
|
{
|
|
DECLARE_REG(u32, id, ctxt, 1);
|
|
u64 prop = 0;
|
|
int ret = 0;
|
|
|
|
if (!ffa_call_supported(id)) {
|
|
ret = FFA_RET_NOT_SUPPORTED;
|
|
goto out_handled;
|
|
}
|
|
|
|
switch (id) {
|
|
case FFA_MEM_SHARE:
|
|
case FFA_FN64_MEM_SHARE:
|
|
case FFA_MEM_LEND:
|
|
case FFA_FN64_MEM_LEND:
|
|
ret = FFA_RET_SUCCESS;
|
|
prop = 0; /* No support for dynamic buffers */
|
|
goto out_handled;
|
|
default:
|
|
return false;
|
|
}
|
|
|
|
out_handled:
|
|
ffa_to_smccc_res_prop(res, ret, prop);
|
|
return true;
|
|
}
|
|
|
|
static void do_ffa_guest_features(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt)
|
|
{
|
|
DECLARE_REG(u32, id, ctxt, 1);
|
|
u64 prop = 0;
|
|
int ret;
|
|
|
|
switch (id) {
|
|
case FFA_MEM_SHARE:
|
|
case FFA_FN64_MEM_SHARE:
|
|
case FFA_MEM_LEND:
|
|
case FFA_FN64_MEM_LEND:
|
|
case FFA_RX_RELEASE:
|
|
ret = FFA_RET_SUCCESS;
|
|
goto out_handled;
|
|
case FFA_RXTX_MAP:
|
|
case FFA_FN64_RXTX_MAP:
|
|
ret = FFA_RET_SUCCESS;
|
|
if (PAGE_SIZE == SZ_4K)
|
|
prop = FFA_FEAT_RXTX_MIN_SZ_4K;
|
|
else if (PAGE_SIZE == SZ_64K)
|
|
prop = FFA_FEAT_RXTX_MIN_SZ_64K;
|
|
else if (PAGE_SIZE == SZ_16K)
|
|
prop = FFA_FEAT_RXTX_MIN_SZ_16K;
|
|
else /* prop == b'11 is reserved per DEN0077A v1.3 ALP1 */
|
|
ret = FFA_RET_NOT_SUPPORTED;
|
|
goto out_handled;
|
|
default:
|
|
ret = FFA_RET_NOT_SUPPORTED;
|
|
goto out_handled;
|
|
}
|
|
|
|
out_handled:
|
|
ffa_to_smccc_res_prop(res, ret, prop);
|
|
}
|
|
|
|
static void do_ffa_part_get_response(struct arm_smccc_1_2_regs *res,
|
|
u32 uuid0, u32 uuid1, u32 uuid2,
|
|
u32 uuid3, u32 flags, struct kvm_ffa_buffers *ffa_buf)
|
|
{
|
|
int ret;
|
|
u32 count, partition_sz, copy_sz;
|
|
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_PARTITION_INFO_GET,
|
|
.a1 = uuid0,
|
|
.a2 = uuid1,
|
|
.a3 = uuid2,
|
|
.a4 = uuid3,
|
|
.a5 = flags,
|
|
}, res);
|
|
|
|
if (res->a0 != FFA_SUCCESS)
|
|
return;
|
|
|
|
count = res->a2;
|
|
if (!count)
|
|
return;
|
|
|
|
if (hyp_ffa_version > FFA_VERSION_1_0) {
|
|
/* Get the number of partitions deployed in the system */
|
|
if (flags & 0x1)
|
|
return;
|
|
|
|
partition_sz = res->a3;
|
|
} else
|
|
/* FFA_VERSION_1_0 lacks the size in the response */
|
|
partition_sz = FFA_1_0_PARTITON_INFO_SZ;
|
|
|
|
copy_sz = partition_sz * count;
|
|
if (copy_sz > KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) {
|
|
ffa_to_smccc_res(res, FFA_RET_ABORTED);
|
|
return;
|
|
}
|
|
|
|
ret = parse_vm_availability_resp(partition_sz, count);
|
|
if (ret)
|
|
ffa_to_smccc_res(res, ret);
|
|
|
|
if (ffa_buf)
|
|
memcpy(ffa_buf->rx, hyp_buffers.rx, copy_sz);
|
|
}
|
|
|
|
static int hyp_ffa_post_init(void)
|
|
{
|
|
size_t min_rxtx_sz;
|
|
struct arm_smccc_1_2_regs res;
|
|
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
|
|
.a0 = FFA_ID_GET,
|
|
}, &res);
|
|
if (res.a0 != FFA_SUCCESS)
|
|
return -EOPNOTSUPP;
|
|
|
|
if (res.a2 != HOST_FFA_ID)
|
|
return -EINVAL;
|
|
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs){
|
|
.a0 = FFA_FEATURES,
|
|
.a1 = FFA_FN64_RXTX_MAP,
|
|
}, &res);
|
|
if (res.a0 != FFA_SUCCESS)
|
|
return -EOPNOTSUPP;
|
|
|
|
switch (res.a2 & FFA_FEAT_RXTX_MIN_SZ_MASK) {
|
|
case FFA_FEAT_RXTX_MIN_SZ_4K:
|
|
min_rxtx_sz = SZ_4K;
|
|
break;
|
|
case FFA_FEAT_RXTX_MIN_SZ_16K:
|
|
min_rxtx_sz = SZ_16K;
|
|
break;
|
|
case FFA_FEAT_RXTX_MIN_SZ_64K:
|
|
min_rxtx_sz = SZ_64K;
|
|
break;
|
|
default:
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (min_rxtx_sz > PAGE_SIZE)
|
|
return -EOPNOTSUPP;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static void do_ffa_version(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt)
|
|
{
|
|
DECLARE_REG(u32, ffa_req_version, ctxt, 1);
|
|
|
|
if (FFA_MAJOR_VERSION(ffa_req_version) != 1) {
|
|
res->a0 = FFA_RET_NOT_SUPPORTED;
|
|
return;
|
|
}
|
|
|
|
hyp_spin_lock(&version_lock);
|
|
if (has_version_negotiated) {
|
|
if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version))
|
|
res->a0 = FFA_RET_NOT_SUPPORTED;
|
|
else
|
|
res->a0 = hyp_ffa_version;
|
|
goto unlock;
|
|
}
|
|
|
|
/*
|
|
* If the client driver tries to downgrade the version, we need to ask
|
|
* first if TEE supports it.
|
|
*/
|
|
if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) {
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_VERSION,
|
|
.a1 = ffa_req_version,
|
|
}, res);
|
|
if (res->a0 == FFA_RET_NOT_SUPPORTED)
|
|
goto unlock;
|
|
|
|
hyp_ffa_version = ffa_req_version;
|
|
}
|
|
|
|
if (hyp_ffa_post_init()) {
|
|
res->a0 = FFA_RET_NOT_SUPPORTED;
|
|
} else {
|
|
/*
|
|
* Ensure that the write to `has_version_negotiated` is visible
|
|
* to other CPUs after the previous operations.
|
|
*/
|
|
smp_store_release(&has_version_negotiated, true);
|
|
res->a0 = hyp_ffa_version;
|
|
}
|
|
unlock:
|
|
hyp_spin_unlock(&version_lock);
|
|
}
|
|
|
|
static void do_ffa_guest_version(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(u32, ffa_req_version, ctxt, 1);
|
|
|
|
if (FFA_MAJOR_VERSION(ffa_req_version) != 1) {
|
|
res->a0 = FFA_RET_NOT_SUPPORTED;
|
|
return;
|
|
}
|
|
|
|
hyp_spin_lock(&version_lock);
|
|
|
|
/* No backwards compatibility for you, please update your guest drivers*/
|
|
if (FFA_MINOR_VERSION(ffa_req_version) < FFA_MINOR_VERSION(hyp_ffa_version)) {
|
|
res->a0 = FFA_RET_NOT_SUPPORTED;
|
|
goto unlock;
|
|
}
|
|
|
|
res->a0 = hyp_ffa_version;
|
|
unlock:
|
|
hyp_spin_unlock(&version_lock);
|
|
}
|
|
|
|
static void do_ffa_part_get(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
struct pkvm_hyp_vcpu *hyp_vcpu)
|
|
{
|
|
DECLARE_REG(u32, uuid0, ctxt, 1);
|
|
DECLARE_REG(u32, uuid1, ctxt, 2);
|
|
DECLARE_REG(u32, uuid2, ctxt, 3);
|
|
DECLARE_REG(u32, uuid3, ctxt, 4);
|
|
DECLARE_REG(u32, flags, ctxt, 5);
|
|
struct kvm_ffa_buffers *ffa_buf;
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_buf = ffa_get_buffers(hyp_vcpu);
|
|
if (!ffa_buf->rx) {
|
|
ffa_to_smccc_res(res, FFA_RET_BUSY);
|
|
goto out_unlock;
|
|
}
|
|
|
|
do_ffa_part_get_response(res, uuid0, uuid1, uuid2, uuid3, flags, ffa_buf);
|
|
out_unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
}
|
|
|
|
static void do_ffa_direct_msg(struct arm_smccc_1_2_regs *res,
|
|
struct kvm_cpu_context *ctxt,
|
|
u64 vm_handle)
|
|
{
|
|
DECLARE_REG(u32, endp, ctxt, 1);
|
|
|
|
struct arm_smccc_1_2_regs *args = (void *)&ctxt->regs.regs[0];
|
|
|
|
if (FIELD_GET(FFA_SRC_ENDPOINT_MASK, endp) != vm_handle) {
|
|
ffa_to_smccc_error(res, FFA_RET_INVALID_PARAMETERS);
|
|
return;
|
|
}
|
|
|
|
nvhe_arm_smccc_1_2_smc(args, res);
|
|
}
|
|
|
|
static int kvm_host_ffa_signal_availability(void)
|
|
{
|
|
int ret;
|
|
struct arm_smccc_1_2_regs res;
|
|
|
|
/*
|
|
* Map our hypervisor buffers into the SPMD before mapping and
|
|
* pinning the host buffers in our own address space.
|
|
*/
|
|
ret = ffa_map_hyp_buffers((KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE) / FFA_PAGE_SIZE);
|
|
if (ret)
|
|
return ffa_to_linux_errno(ret);
|
|
|
|
do_ffa_part_get_response(&res, 0, 0, 0, 0, 0, NULL);
|
|
if (res.a0 != FFA_SUCCESS)
|
|
return ffa_to_linux_errno(ret);
|
|
|
|
ffa_rx_release(&res);
|
|
|
|
return kvm_notify_vm_availability(HOST_FFA_ID, &host_buffers, FFA_VM_CREATION_MSG);
|
|
}
|
|
|
|
bool kvm_host_ffa_handler(struct kvm_cpu_context *host_ctxt, u32 func_id)
|
|
{
|
|
struct arm_smccc_1_2_regs res;
|
|
int ret;
|
|
|
|
/*
|
|
* There's no way we can tell what a non-standard SMC call might
|
|
* be up to. Ideally, we would terminate these here and return
|
|
* an error to the host, but sadly devices make use of custom
|
|
* firmware calls for things like power management, debugging,
|
|
* RNG access and crash reporting.
|
|
*
|
|
* Given that the architecture requires us to trust EL3 anyway,
|
|
* we forward unrecognised calls on under the assumption that
|
|
* the firmware doesn't expose a mechanism to access arbitrary
|
|
* non-secure memory. Short of a per-device table of SMCs, this
|
|
* is the best we can do.
|
|
*/
|
|
if (!is_ffa_call(func_id))
|
|
return false;
|
|
|
|
if (func_id != FFA_VERSION &&
|
|
!smp_load_acquire(&has_version_negotiated)) {
|
|
ffa_to_smccc_error(&res, FFA_RET_INVALID_PARAMETERS);
|
|
goto out_handled;
|
|
}
|
|
|
|
/*
|
|
* Notify TZ of host VM creation immediately
|
|
* before handling the first non-version SMC/HVC
|
|
*/
|
|
if (func_id != FFA_VERSION && !has_host_signalled) {
|
|
ret = kvm_host_ffa_signal_availability();
|
|
if (!ret)
|
|
/*
|
|
* Ensure that the write to `has_host_signalled` is visible
|
|
* to other CPUs after the previous operations.
|
|
*/
|
|
has_host_signalled = true;
|
|
else if (ret == -EAGAIN || ret == -EINTR) {
|
|
/*
|
|
* Don't retry with interrupts masked as we will spin
|
|
* forever.
|
|
*/
|
|
if (host_ctxt->regs.pstate & PSR_I_BIT) {
|
|
ffa_to_smccc_error(&res, FFA_RET_DENIED);
|
|
goto out_handled;
|
|
}
|
|
|
|
/* Go back to the host and replay the last instruction */
|
|
write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR);
|
|
return true;
|
|
}
|
|
}
|
|
|
|
switch (func_id) {
|
|
case FFA_FEATURES:
|
|
if (!do_ffa_features(&res, host_ctxt))
|
|
return false;
|
|
goto out_handled;
|
|
/* Memory management */
|
|
case FFA_FN64_RXTX_MAP:
|
|
do_ffa_rxtx_map(&res, host_ctxt, NULL);
|
|
goto out_handled;
|
|
case FFA_RXTX_UNMAP:
|
|
do_ffa_rxtx_unmap(&res, host_ctxt, NULL);
|
|
goto out_handled;
|
|
case FFA_MEM_SHARE:
|
|
case FFA_FN64_MEM_SHARE:
|
|
do_ffa_mem_xfer(FFA_FN64_MEM_SHARE, &res, host_ctxt, NULL);
|
|
goto out_handled;
|
|
case FFA_MEM_RECLAIM:
|
|
do_ffa_mem_reclaim(&res, host_ctxt, NULL);
|
|
goto out_handled;
|
|
case FFA_MEM_LEND:
|
|
case FFA_FN64_MEM_LEND:
|
|
do_ffa_mem_xfer(FFA_FN64_MEM_LEND, &res, host_ctxt, NULL);
|
|
goto out_handled;
|
|
case FFA_MEM_FRAG_TX:
|
|
do_ffa_mem_frag_tx(&res, host_ctxt, NULL);
|
|
goto out_handled;
|
|
case FFA_VERSION:
|
|
do_ffa_version(&res, host_ctxt);
|
|
goto out_handled;
|
|
case FFA_PARTITION_INFO_GET:
|
|
do_ffa_part_get(&res, host_ctxt, NULL);
|
|
goto out_handled;
|
|
case FFA_RX_RELEASE:
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_rx_release(&res);
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
goto out_handled;
|
|
case FFA_ID_GET:
|
|
ffa_to_smccc_res_prop(&res, FFA_RET_SUCCESS, HOST_FFA_ID);
|
|
goto out_handled;
|
|
case FFA_MSG_SEND_DIRECT_REQ2:
|
|
if (!ffa_call_supported(func_id))
|
|
goto out_not_supported;
|
|
fallthrough;
|
|
case FFA_MSG_SEND_DIRECT_REQ:
|
|
case FFA_FN64_MSG_SEND_DIRECT_REQ:
|
|
do_ffa_direct_msg(&res, host_ctxt, HOST_FFA_ID);
|
|
goto out_handled;
|
|
}
|
|
|
|
if (ffa_call_supported(func_id))
|
|
return false; /* Pass through */
|
|
|
|
out_not_supported:
|
|
ffa_to_smccc_error(&res, FFA_RET_NOT_SUPPORTED);
|
|
out_handled:
|
|
ffa_set_retval(host_ctxt, &res);
|
|
return true;
|
|
}
|
|
|
|
bool kvm_guest_ffa_handler(struct pkvm_hyp_vcpu *hyp_vcpu, u64 *exit_code)
|
|
{
|
|
struct kvm_vcpu *vcpu = &hyp_vcpu->vcpu;
|
|
struct kvm_cpu_context *ctxt = &vcpu->arch.ctxt;
|
|
struct arm_smccc_1_2_regs res;
|
|
struct kvm_hyp_req *req;
|
|
int ret;
|
|
|
|
DECLARE_REG(u64, func_id, ctxt, 0);
|
|
|
|
if (!is_ffa_call(func_id) || !VM_FFA_SUPPORTED(vcpu) ||
|
|
!smp_load_acquire(&has_version_negotiated)) {
|
|
smccc_set_retval(vcpu, SMCCC_RET_NOT_SUPPORTED, 0, 0, 0);
|
|
return true;
|
|
}
|
|
|
|
switch (func_id) {
|
|
case FFA_FEATURES:
|
|
do_ffa_guest_features(&res, ctxt);
|
|
goto out_guest;
|
|
case FFA_VERSION:
|
|
do_ffa_guest_version(&res, ctxt, hyp_vcpu);
|
|
goto out_guest;
|
|
case FFA_FN64_RXTX_MAP:
|
|
ret = do_ffa_rxtx_guest_map(ctxt, hyp_vcpu);
|
|
break;
|
|
case FFA_RXTX_UNMAP:
|
|
do_ffa_rxtx_unmap(&res, ctxt, hyp_vcpu);
|
|
goto out_guest;
|
|
case FFA_MEM_RECLAIM:
|
|
do_ffa_mem_reclaim(&res, ctxt, hyp_vcpu);
|
|
goto out_guest;
|
|
case FFA_MEM_SHARE:
|
|
case FFA_FN64_MEM_SHARE:
|
|
ret = do_ffa_mem_xfer(FFA_FN64_MEM_SHARE, &res, ctxt, hyp_vcpu);
|
|
if (!ret)
|
|
goto out_guest;
|
|
break;
|
|
case FFA_MEM_LEND:
|
|
case FFA_FN64_MEM_LEND:
|
|
ret = do_ffa_mem_xfer(FFA_FN64_MEM_LEND, &res, ctxt, hyp_vcpu);
|
|
if (!ret)
|
|
goto out_guest;
|
|
break;
|
|
case FFA_ID_GET:
|
|
ffa_to_smccc_res_prop(&res, FFA_RET_SUCCESS, hyp_vcpu_to_ffa_handle(hyp_vcpu));
|
|
goto out_guest;
|
|
case FFA_PARTITION_INFO_GET:
|
|
do_ffa_part_get(&res, ctxt, hyp_vcpu);
|
|
goto out_guest;
|
|
case FFA_RX_RELEASE:
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ffa_rx_release(&res);
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
goto out_guest;
|
|
case FFA_MSG_SEND_DIRECT_REQ2:
|
|
if (!ffa_call_supported(func_id)) {
|
|
ret = -EOPNOTSUPP;
|
|
goto out_guest_with_ret;
|
|
}
|
|
fallthrough;
|
|
case FFA_MSG_SEND_DIRECT_REQ:
|
|
case FFA_FN64_MSG_SEND_DIRECT_REQ:
|
|
do_ffa_direct_msg(&res, ctxt, hyp_vcpu_to_ffa_handle(hyp_vcpu));
|
|
goto out_guest;
|
|
default:
|
|
ret = -EOPNOTSUPP;
|
|
break;
|
|
}
|
|
|
|
switch (ret) {
|
|
case -ENOMEM:
|
|
if (hyp_alloc_errno() != -ENOMEM) {
|
|
/* Only hyp_alloc() can raise -ENOMEM. Let the HVC fail */
|
|
ret = hyp_alloc_errno();
|
|
goto out_guest_with_ret;
|
|
}
|
|
|
|
req = pkvm_hyp_req_reserve(hyp_vcpu, KVM_HYP_REQ_TYPE_MEM);
|
|
if (!req)
|
|
goto out_guest_with_ret;
|
|
|
|
req->mem.dest = REQ_MEM_DEST_HYP_ALLOC;
|
|
req->mem.nr_pages = hyp_alloc_missing_donations();
|
|
fallthrough;
|
|
case -EFAULT:
|
|
fallthrough;
|
|
case -E2BIG:
|
|
/*
|
|
* EFAULT and E2BIG have already been handled earlier with
|
|
* __handle_mem_protect_err(). Go back to the host and replay the last guest
|
|
* instruction.
|
|
*/
|
|
write_sysreg_el2(read_sysreg_el2(SYS_ELR) - 4, SYS_ELR);
|
|
*exit_code = ARM_EXCEPTION_HYP_REQ;
|
|
return false;
|
|
}
|
|
|
|
out_guest_with_ret:
|
|
ffa_to_smccc_res(&res, linux_errno_to_ffa(ret));
|
|
out_guest:
|
|
ffa_set_retval(ctxt, &res);
|
|
return true;
|
|
}
|
|
|
|
static void kvm_guest_try_reclaim_transfer(struct ffa_mem_transfer *transfer,
|
|
struct pkvm_hyp_vm *vm)
|
|
{
|
|
struct ffa_translation *translation, *tmp;
|
|
struct arm_smccc_1_2_regs res;
|
|
|
|
ffa_mem_reclaim(&res, HANDLE_LOW(transfer->ffa_handle),
|
|
HANDLE_HIGH(transfer->ffa_handle), 0);
|
|
if (res.a0 != FFA_SUCCESS)
|
|
return;
|
|
|
|
list_for_each_entry_safe(translation, tmp, &transfer->translations, node) {
|
|
WARN_ON(__pkvm_guest_unshare_ffa_page(vm->vcpus[0], translation->ipa));
|
|
list_del(&translation->node);
|
|
hyp_free(translation);
|
|
}
|
|
|
|
list_del(&transfer->node);
|
|
hyp_free(transfer);
|
|
}
|
|
|
|
int kvm_dying_guest_reclaim_ffa_resources(struct pkvm_hyp_vm *vm)
|
|
{
|
|
struct kvm_ffa_buffers *ffa_buf = &vm->ffa_buf;
|
|
struct ffa_mem_transfer *transfer;
|
|
int ret = 0;
|
|
|
|
if (!vm->kvm.arch.pkvm.ffa_support)
|
|
return 0;
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
if (!ffa_buf->tx && !ffa_buf->rx)
|
|
goto unlock;
|
|
|
|
if (list_empty(&ffa_buf->xfer_list)) {
|
|
/* XXX - needs an explicit rxtx unmap call ? */
|
|
if (ffa_buf->tx) {
|
|
WARN_ON(__pkvm_guest_unshare_hyp_page(vm->vcpus[0], ffa_buf->tx_ipa));
|
|
ffa_buf->tx = NULL;
|
|
}
|
|
if (ffa_buf->rx) {
|
|
WARN_ON(__pkvm_guest_unshare_hyp_page(vm->vcpus[0], ffa_buf->rx_ipa));
|
|
ffa_buf->rx = NULL;
|
|
}
|
|
goto unlock;
|
|
}
|
|
|
|
transfer = list_first_entry(&ffa_buf->xfer_list, typeof(*transfer), node);
|
|
kvm_guest_try_reclaim_transfer(transfer, vm);
|
|
ret = -EAGAIN;
|
|
|
|
unlock:
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
int kvm_guest_notify_availability(u32 ffa_handle, struct kvm_ffa_buffers *ffa_buf, bool is_dying)
|
|
{
|
|
int ret;
|
|
|
|
hyp_spin_lock(&kvm_ffa_hyp_lock);
|
|
ret = kvm_notify_vm_availability(ffa_handle, ffa_buf,
|
|
is_dying ? FFA_VM_DESTRUCTION_MSG : FFA_VM_CREATION_MSG);
|
|
hyp_spin_unlock(&kvm_ffa_hyp_lock);
|
|
|
|
return ret;
|
|
}
|
|
|
|
u32 ffa_get_hypervisor_version(void)
|
|
{
|
|
u32 version = 0;
|
|
|
|
hyp_spin_lock(&version_lock);
|
|
if (has_version_negotiated)
|
|
version = hyp_ffa_version;
|
|
hyp_spin_unlock(&version_lock);
|
|
|
|
return version;
|
|
}
|
|
|
|
int hyp_ffa_init(void *pages)
|
|
{
|
|
struct arm_smccc_1_2_regs res;
|
|
void *tx, *rx;
|
|
|
|
if (kvm_host_psci_config.smccc_version < ARM_SMCCC_VERSION_1_1)
|
|
return 0;
|
|
|
|
nvhe_arm_smccc_1_2_smc(&(struct arm_smccc_1_2_regs) {
|
|
.a0 = FFA_VERSION,
|
|
.a1 = FFA_VERSION_1_2,
|
|
}, &res);
|
|
if (res.a0 == FFA_RET_NOT_SUPPORTED)
|
|
return 0;
|
|
|
|
/*
|
|
* Firmware returns the maximum supported version of the FF-A
|
|
* implementation. Check that the returned version is
|
|
* backwards-compatible with the hyp according to the rules in DEN0077A
|
|
* v1.1 REL0 13.2.1.
|
|
*
|
|
* Of course, things are never simple when dealing with firmware. v1.1
|
|
* broke ABI with v1.0 on several structures, which is itself
|
|
* incompatible with the aforementioned versioning scheme. The
|
|
* expectation is that v1.x implementations that do not support the v1.0
|
|
* ABI return NOT_SUPPORTED rather than a version number, according to
|
|
* DEN0077A v1.1 REL0 18.6.4.
|
|
*/
|
|
if (FFA_MAJOR_VERSION(res.a0) != 1)
|
|
return -EOPNOTSUPP;
|
|
|
|
if (FFA_MINOR_VERSION(res.a0) < FFA_MINOR_VERSION(FFA_VERSION_1_2))
|
|
hyp_ffa_version = res.a0;
|
|
else
|
|
hyp_ffa_version = FFA_VERSION_1_2;
|
|
|
|
tx = pages;
|
|
pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE;
|
|
rx = pages;
|
|
pages += KVM_FFA_MBOX_NR_PAGES * PAGE_SIZE;
|
|
|
|
if (static_branch_unlikely(&kvm_ffa_unmap_on_lend)) {
|
|
spm_handles = pages;
|
|
pages += KVM_FFA_SPM_HANDLE_NR_PAGES * PAGE_SIZE;
|
|
num_spm_handles = KVM_FFA_SPM_HANDLE_NR_PAGES * PAGE_SIZE /
|
|
sizeof(struct ffa_handle);
|
|
memset(spm_handles, -1, KVM_FFA_SPM_HANDLE_NR_PAGES * PAGE_SIZE);
|
|
}
|
|
|
|
ffa_desc_buf = (struct kvm_ffa_descriptor_buffer) {
|
|
.buf = pages,
|
|
.len = PAGE_SIZE *
|
|
(hyp_ffa_proxy_pages() - (2 * KVM_FFA_MBOX_NR_PAGES)),
|
|
};
|
|
|
|
hyp_buffers = (struct kvm_ffa_buffers) {
|
|
.tx = tx,
|
|
.rx = rx,
|
|
};
|
|
|
|
version_lock = __HYP_SPIN_LOCK_UNLOCKED;
|
|
INIT_LIST_HEAD(&host_buffers.xfer_list);
|
|
|
|
return 0;
|
|
}
|