From 785e35353b522b72d22038a0deecb981635f1db0 Mon Sep 17 00:00:00 2001 From: yxk Date: Wed, 17 Dec 2025 23:28:13 +0800 Subject: [PATCH] kvm/rme: CCA supports DA. --- arch/arm64/configs/openeuler_defconfig | 1 + arch/arm64/include/asm/kvm_rme.h | 17 +- arch/arm64/include/asm/kvm_rme_da.h | 56 + arch/arm64/include/asm/mem_encrypt.h | 4 + arch/arm64/include/asm/realm_guest.h | 16 + arch/arm64/include/asm/rme_pmu.h | 90 ++ arch/arm64/include/asm/rmi_cmds.h | 780 +++++++++-- arch/arm64/include/asm/rmi_smc.h | 52 +- arch/arm64/include/asm/rsi.h | 22 + arch/arm64/include/asm/rsi_cmds.h | 90 +- arch/arm64/include/asm/rsi_smc.h | 8 + arch/arm64/include/uapi/asm/kvm.h | 15 +- arch/arm64/kernel/Makefile | 2 +- arch/arm64/kernel/realm_guest.c | 50 + arch/arm64/kernel/rsi.c | 6 + arch/arm64/kvm/Kconfig | 8 + arch/arm64/kvm/Makefile | 2 + arch/arm64/kvm/arm.c | 14 + arch/arm64/kvm/mmu.c | 10 +- arch/arm64/kvm/rme-da.c | 794 +++++++++++ arch/arm64/kvm/rme-exit.c | 16 + arch/arm64/kvm/rme-pmu.c | 30 + arch/arm64/kvm/rme.c | 295 +++- drivers/base/core.c | 2 + drivers/base/platform-msi.c | 2 +- drivers/iommu/Kconfig | 9 + drivers/iommu/Makefile | 1 + drivers/iommu/arm/arm-smmu-v3/Makefile | 1 + drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.c | 1215 +++++++++++++++++ drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.h | 83 ++ .../arm/arm-smmu-v3/arm-smmu-v3-iommufd.c | 10 + .../iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c | 5 + drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c | 99 ++ drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h | 85 ++ drivers/iommu/io-pgtable-realm.c | 1019 ++++++++++++++ drivers/iommu/io-pgtable.c | 6 + drivers/iommu/iommu.c | 12 + drivers/iommu/iommufd/hw_pagetable.c | 13 + drivers/iommu/iommufd/ioas.c | 20 + drivers/iommu/iommufd/iommufd_private.h | 2 + drivers/iommu/iommufd/main.c | 3 + drivers/iommu/iommufd/vfio_compat.c | 9 +- drivers/irqchip/irq-gic-v3-its.c | 11 +- drivers/pci/msi/msi.c | 17 +- drivers/pci/msi/msi.h | 15 +- drivers/pci/pci.c | 9 + drivers/vfio/pci/vfio_pci.c | 38 + drivers/vfio/pci/vfio_pci_core.c | 10 + drivers/vfio/vfio_iommu_type1.c | 17 +- drivers/virtio/virtio_mmio.c | 3 + drivers/virtio/virtio_pci_common.c | 4 +- include/linux/io-pgtable.h | 13 + include/linux/iommu.h | 7 + include/uapi/linux/iommufd.h | 9 +- include/uapi/linux/kvm.h | 21 + include/uapi/linux/vfio.h | 3 + kernel/dma/pool.c | 8 + kernel/dma/swiotlb.c | 3 + 58 files changed, 5026 insertions(+), 136 deletions(-) create mode 100644 arch/arm64/include/asm/kvm_rme_da.h create mode 100644 arch/arm64/include/asm/realm_guest.h create mode 100644 arch/arm64/include/asm/rme_pmu.h create mode 100644 arch/arm64/kernel/realm_guest.c create mode 100644 arch/arm64/kvm/rme-da.c create mode 100644 arch/arm64/kvm/rme-pmu.c create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.c create mode 100644 drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.h create mode 100644 drivers/iommu/io-pgtable-realm.c diff --git a/arch/arm64/configs/openeuler_defconfig b/arch/arm64/configs/openeuler_defconfig index 8a7983c944d2..44e2c0d73203 100644 --- a/arch/arm64/configs/openeuler_defconfig +++ b/arch/arm64/configs/openeuler_defconfig @@ -6672,6 +6672,7 @@ CONFIG_ARM_SMMU_QCOM=y # CONFIG_ARM_SMMU_QCOM_DEBUG is not set CONFIG_ARM_SMMU_V3=y CONFIG_ARM_SMMU_V3_SVA=y +CONFIG_HISI_CCA_DA=y # CONFIG_ARM_SMMU_V3_PM is not set CONFIG_ARM_SMMU_V3_HTTU=y CONFIG_ARM_SMMU_V3_ECMDQ=y diff --git a/arch/arm64/include/asm/kvm_rme.h b/arch/arm64/include/asm/kvm_rme.h index 35f2771602c4..1abe39b9d64d 100644 --- a/arch/arm64/include/asm/kvm_rme.h +++ b/arch/arm64/include/asm/kvm_rme.h @@ -8,6 +8,7 @@ #include #include +#include /** * enum realm_state - State of a Realm @@ -54,6 +55,7 @@ enum realm_state { * @num_aux: The number of auxiliary pages required by the RMM * @vmid: VMID to be used by the RMM for the realm * @ia_bits: Number of valid Input Address bits in the IPA + * @undelegate_list: Undelegate region waiting list */ struct realm { enum realm_state state; @@ -66,6 +68,11 @@ struct realm { unsigned int ia_bits; #ifndef __GENKSYMS__ bool is_ccal; + struct list_head undelegate_list; + bool delay_undelegate; +#ifdef CONFIG_HISI_CCA_DA + struct list_head rdev_list; +#endif #endif }; @@ -117,12 +124,20 @@ int _handle_rec_exit(struct kvm_vcpu *vcpu, int rec_run_status); void kvm_realm_unmap_range(struct kvm *kvm, unsigned long ipa, unsigned long size, - bool unmap_private); + bool unmap_private, + bool delay_undelegate); int realm_map_protected(struct realm *realm, unsigned long base_ipa, kvm_pfn_t pfn, unsigned long size, struct kvm_mmu_memory_cache *memcache); +#ifdef CONFIG_HISI_CCA_DA +int realm_map_mmio_protected(struct realm *realm, + unsigned long ipa, + kvm_pfn_t pfn, + unsigned long map_size, + struct kvm_mmu_memory_cache *memcache); +#endif int realm_map_non_secure(struct realm *realm, unsigned long ipa, kvm_pfn_t pfn, diff --git a/arch/arm64/include/asm/kvm_rme_da.h b/arch/arm64/include/asm/kvm_rme_da.h new file mode 100644 index 000000000000..9ed5c852a367 --- /dev/null +++ b/arch/arm64/include/asm/kvm_rme_da.h @@ -0,0 +1,56 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __ASM_KVM_RME_DA_H +#define __ASM_KVM_RME_DA_H + +#include +#include + +#define MAX_DEV_PER_PORT 256 +struct rmi_dev_assign_params { + /* BDF of PCIe root bus, F=0. BD are used to calculate APB base and port number. */ + uint16_t root_bdf; + uint16_t num_dev; /* number of attachable devices */ + uint16_t last_batch; /* Is this the last batch in the sequence */ + uint16_t rsvd; /* padding for 64-bit alignment */ + uint16_t devs[MAX_DEV_PER_PORT]; /* BDF of each attachable device */ +}; + +struct rdev_node { + uint16_t dev_bdf; + struct list_head list; +}; + +bool is_support_rme(void); + +int kvm_rme_dev_assign(struct pci_dev *pdev, struct kvm *kvm); +int kvm_rme_dev_attach(struct pci_dev *pdev, struct kvm *kvm); +void kvm_rme_dev_unassign(struct pci_dev *pdev, struct kvm *kvm); +void kvm_rme_dev_detach(struct pci_dev *pdev, struct kvm *kvm); +int kvm_arm_vcpu_rme_dev_validate(struct kvm_vcpu *vcpu, struct kvm_arm_rme_dev_validate *args); +void kvm_complete_dev_validate(struct kvm_vcpu *vcpu); +int realm_attach_devs(struct realm *realm); + +int realm_pcipc_ns_add(const struct pci_device_id *id_table); +void realm_pcipc_ns_remove(const struct pci_device_id *id_table); +bool is_realm_pcipc_ns(struct device *dev); + +bool rme_dev_pci_read_msi_msg(struct msi_desc *desc, struct msi_msg *msg); +bool rme_dev_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg); +bool rme_dev_msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc); +bool rme_dev_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl); +bool rme_dev_pci_msix_mask(struct msi_desc *desc); +bool rme_dev_msix_mask_all(struct pci_dev *dev, int tsize); + +int ccada_init_mem_region(struct pci_dev *pdev, int bar); + +#ifdef CONFIG_HISI_CCA_DA +u32 readl_cca_hook(volatile void __iomem *addr, struct pci_dev *pdev); +void writel_cca_hook(u32 val, volatile void __iomem *addr, struct pci_dev *pdev); +void __raw_writel_cca_hook(u32 val, volatile void __iomem *addr, struct pci_dev *pdev); +#else +#define __raw_writel_cca_hook(v, a, p) __raw_writel(v, a) +#define writel_cca_hook(v, a, p) writel(v, a) +#define readl_cca_hook(a, p) readl(a) +#endif + +#endif diff --git a/arch/arm64/include/asm/mem_encrypt.h b/arch/arm64/include/asm/mem_encrypt.h index a2a1eeb36d4b..b048a6362a2b 100644 --- a/arch/arm64/include/asm/mem_encrypt.h +++ b/arch/arm64/include/asm/mem_encrypt.h @@ -18,7 +18,11 @@ int realm_register_memory_enc_ops(void); static inline bool force_dma_unencrypted(struct device *dev) { +#ifdef CONFIG_HISI_CCA_DA + return false; +#else return is_realm_world(); +#endif } /* diff --git a/arch/arm64/include/asm/realm_guest.h b/arch/arm64/include/asm/realm_guest.h new file mode 100644 index 000000000000..fde5c67e90ed --- /dev/null +++ b/arch/arm64/include/asm/realm_guest.h @@ -0,0 +1,16 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (C) 2025. Huawei Technologies Co., Ltd. All rights reserved. + */ +#ifndef __REALM_GUEST_H +#define __REALM_GUEST_H + +void enable_swiotlb_for_realm_dev(struct device *dev, bool enable); + +void realm_guest_init(void); + +struct page *realm_alloc_shared_pages(gfp_t gfp, unsigned int order); + +void realm_free_shared_pages(void *addr, int order); + +#endif /* __REALM_GUEST_H */ \ No newline at end of file diff --git a/arch/arm64/include/asm/rme_pmu.h b/arch/arm64/include/asm/rme_pmu.h new file mode 100644 index 000000000000..942d80e0b65c --- /dev/null +++ b/arch/arm64/include/asm/rme_pmu.h @@ -0,0 +1,90 @@ +/* SPDX-License-Identifier: GPL-2.0 */ + +#ifndef __ASM_RME_PMU_H +#define __ASM_RME_PMU_H + +#include +#include +#include + +#ifdef CONFIG_CCA_RME_PMU + +#define RMI_SMC_MIN_VAL 0x150 +#define RSI_SMC_MIN_VAL 0x190 +#define RMI_IDX(smc_val) (ARM_SMCCC_FUNC_NUM(smc_val) - RMI_SMC_MIN_VAL) +#define RSI_IDX(smc_val) (ARM_SMCCC_FUNC_NUM(smc_val) - RSI_SMC_MIN_VAL) +#define SMC_RECORD_NUM (RSI_SMC_MIN_VAL - RMI_SMC_MIN_VAL - 1) + +#ifndef RME_PERF_STRUCT_RECORDER +#define RME_PERF_STRUCT_RECORDER + +struct recorder { + unsigned long times; + unsigned long duration; +}; + +#endif /* !RME_PERF_STRUCT_RECORDER */ + +struct rme_perf_report { + unsigned int smc_num; + unsigned long cntfrq; + struct recorder smc_recorder[SMC_RECORD_NUM]; +}; + +struct rme_perf_record_unit { + bool start_record; + struct mutex lock; + struct rme_perf_report report; +}; + +struct rme_perf_record_unit *get_rme_pmu(void); +void rme_perf_init(void); +void rme_perf_record(int idx, unsigned long duration); + +#define RMI_PERF_INIT() rme_perf_init() +#define RMI_PERF_START(x) unsigned long x = arch_timer_read_cntpct_el0() +#define RMI_PERF_RECORD(func_id, duration) rme_perf_record(func_id, duration) + +#define RSI_PERF_INIT() rme_perf_init() +#define RSI_PERF_START(x) unsigned long x = arch_timer_read_cntpct_el0() +#define RSI_PERF_RECORD(func_id, duration) rme_perf_record(func_id, duration) + +static inline void invoke_rmi_smc(const struct arm_smccc_1_2_regs *args, + struct arm_smccc_1_2_regs *res) +{ + int rmi_idx; + + rmi_idx = RMI_IDX(args->a0); + RMI_PERF_START(start_cnt); + arm_smccc_1_2_smc(args, res); + RMI_PERF_RECORD(rmi_idx, arch_timer_read_cntpct_el0() - start_cnt); +} + +static inline void invoke_rsi_smc(const struct arm_smccc_1_2_regs *args, + struct arm_smccc_1_2_regs *res) +{ + int rsi_idx; + + rsi_idx = RSI_IDX(args->a0); + RMI_PERF_START(start_cnt); + arm_smccc_1_2_smc(args, res); + RMI_PERF_RECORD(rsi_idx, arch_timer_read_cntpct_el0() - start_cnt); +} + +#else /* !CONFIG_CCA_RME_PMU */ + +static inline void invoke_rmi_smc(const struct arm_smccc_1_2_regs *args, + struct arm_smccc_1_2_regs *res) +{ + return arm_smccc_1_2_smc(args, res); +} + +static inline void invoke_rsi_smc(const struct arm_smccc_1_2_regs *args, + struct arm_smccc_1_2_regs *res) +{ + return arm_smccc_1_2_smc(args, res); +} + +#endif /* !CONFIG_CCA_RME_PMU */ + +#endif /* __ASM_RME_PMU_H */ diff --git a/arch/arm64/include/asm/rmi_cmds.h b/arch/arm64/include/asm/rmi_cmds.h index 8dc752ca9a74..d172a4acbd72 100644 --- a/arch/arm64/include/asm/rmi_cmds.h +++ b/arch/arm64/include/asm/rmi_cmds.h @@ -9,6 +9,7 @@ #include #include +#include struct rtt_entry { unsigned long walk_level; @@ -33,12 +34,14 @@ static inline int rmi_data_create(unsigned long rd, unsigned long data, unsigned long ipa, unsigned long src, unsigned long flags) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_DATA_CREATE, + rd, data, ipa, src, flags + }; - arm_smccc_1_1_invoke(SMC_RMI_DATA_CREATE, rd, data, ipa, src, - flags, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -53,11 +56,14 @@ static inline int rmi_data_create_unknown(unsigned long rd, unsigned long data, unsigned long ipa) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_DATA_CREATE_UNKNOWN, + rd, data, ipa + }; - arm_smccc_1_1_invoke(SMC_RMI_DATA_CREATE_UNKNOWN, rd, data, ipa, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -77,16 +83,19 @@ static inline int rmi_data_destroy(unsigned long rd, unsigned long ipa, unsigned long *data_out, unsigned long *top_out) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_DATA_DESTROY, + rd, ipa + }; - arm_smccc_1_1_invoke(SMC_RMI_DATA_DESTROY, rd, ipa, &res); + invoke_rmi_smc(®s, ®s); if (data_out) - *data_out = res.a1; + *data_out = regs.a1; if (top_out) - *top_out = res.a2; + *top_out = regs.a2; - return res.a0; + return regs.a0; } /** @@ -98,13 +107,16 @@ static inline int rmi_data_destroy(unsigned long rd, unsigned long ipa, */ static inline int rmi_features(unsigned long index, unsigned long *out) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_FEATURES, + index + }; - arm_smccc_1_1_invoke(SMC_RMI_FEATURES, index, &res); + invoke_rmi_smc(®s, ®s); if (out) - *out = res.a1; - return res.a0; + *out = regs.a1; + return regs.a0; } /** @@ -117,11 +129,14 @@ static inline int rmi_features(unsigned long index, unsigned long *out) */ static inline int rmi_granule_delegate(unsigned long phys) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_GRANULE_DELEGATE, + phys + }; - arm_smccc_1_1_invoke(SMC_RMI_GRANULE_DELEGATE, phys, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -135,11 +150,50 @@ static inline int rmi_granule_delegate(unsigned long phys) */ static inline int rmi_granule_undelegate(unsigned long phys) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_GRANULE_UNDELEGATE, + phys + }; - arm_smccc_1_1_invoke(SMC_RMI_GRANULE_UNDELEGATE, phys, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; +} + +#define GRANULE_DELEGATE_SIZE 4096 + +static inline int granule_undelegate_range(unsigned long phys, size_t size) +{ + size_t off; + int ret; + + if (!phys) + return -EINVAL; + + for (off = 0; off < size; off += GRANULE_DELEGATE_SIZE) { + ret = rmi_granule_undelegate(phys + off); + if (ret) + return ret; + } + + return 0; +} + +static inline int granule_delegate_range(unsigned long phys, size_t size) +{ + size_t off; + int ret = 0; + + for (off = 0; off < size; off += GRANULE_DELEGATE_SIZE) { + ret = rmi_granule_delegate(phys + off); + if (ret) + break; + } + + if (ret) + granule_undelegate_range(phys, off - GRANULE_DELEGATE_SIZE); + + return ret; } /** @@ -157,12 +211,14 @@ static inline int rmi_psci_complete(unsigned long calling_rec, unsigned long target_rec, unsigned long status) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_PSCI_COMPLETE, + calling_rec, target_rec, status + }; - arm_smccc_1_1_invoke(SMC_RMI_PSCI_COMPLETE, calling_rec, target_rec, - status, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -176,11 +232,14 @@ static inline int rmi_psci_complete(unsigned long calling_rec, */ static inline int rmi_realm_activate(unsigned long rd) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REALM_ACTIVATE, + rd + }; - arm_smccc_1_1_invoke(SMC_RMI_REALM_ACTIVATE, rd, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -194,11 +253,14 @@ static inline int rmi_realm_activate(unsigned long rd) */ static inline int rmi_realm_create(unsigned long rd, unsigned long params) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REALM_CREATE, + rd, params + }; - arm_smccc_1_1_invoke(SMC_RMI_REALM_CREATE, rd, params, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -211,11 +273,14 @@ static inline int rmi_realm_create(unsigned long rd, unsigned long params) */ static inline int rmi_realm_destroy(unsigned long rd) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REALM_DESTROY, + rd + }; - arm_smccc_1_1_invoke(SMC_RMI_REALM_DESTROY, rd, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -231,13 +296,16 @@ static inline int rmi_realm_destroy(unsigned long rd) */ static inline int rmi_rec_aux_count(unsigned long rd, unsigned long *aux_count) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REC_AUX_COUNT, + rd + }; - arm_smccc_1_1_invoke(SMC_RMI_REC_AUX_COUNT, rd, &res); + invoke_rmi_smc(®s, ®s); if (aux_count) - *aux_count = res.a1; - return res.a0; + *aux_count = regs.a1; + return regs.a0; } /** @@ -254,11 +322,14 @@ static inline int rmi_rec_aux_count(unsigned long rd, unsigned long *aux_count) static inline int rmi_rec_create(unsigned long rd, unsigned long rec, unsigned long params) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REC_CREATE, + rd, rec, params + }; - arm_smccc_1_1_invoke(SMC_RMI_REC_CREATE, rd, rec, params, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -271,11 +342,14 @@ static inline int rmi_rec_create(unsigned long rd, unsigned long rec, */ static inline int rmi_rec_destroy(unsigned long rec) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REC_DESTROY, + rec + }; - arm_smccc_1_1_invoke(SMC_RMI_REC_DESTROY, rec, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -289,11 +363,14 @@ static inline int rmi_rec_destroy(unsigned long rec) */ static inline int rmi_rec_enter(unsigned long rec, unsigned long run_ptr) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_REC_ENTER, + rec, run_ptr + }; - arm_smccc_1_1_invoke(SMC_RMI_REC_ENTER, rec, run_ptr, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -311,11 +388,14 @@ static inline int rmi_rec_enter(unsigned long rec, unsigned long run_ptr) static inline int rmi_rtt_create(unsigned long rd, unsigned long rtt, unsigned long ipa, long level) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_CREATE, + rd, rtt, ipa, level + }; - arm_smccc_1_1_invoke(SMC_RMI_RTT_CREATE, rd, rtt, ipa, level, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -337,16 +417,19 @@ static inline int rmi_rtt_destroy(unsigned long rd, unsigned long *out_rtt, unsigned long *out_top) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_DESTROY, + rd, ipa, level + }; - arm_smccc_1_1_invoke(SMC_RMI_RTT_DESTROY, rd, ipa, level, &res); + invoke_rmi_smc(®s, ®s); if (out_rtt) - *out_rtt = res.a1; + *out_rtt = regs.a1; if (out_top) - *out_top = res.a2; + *out_top = regs.a2; - return res.a0; + return regs.a0; } /** @@ -364,14 +447,17 @@ static inline int rmi_rtt_destroy(unsigned long rd, static inline int rmi_rtt_fold(unsigned long rd, unsigned long ipa, long level, unsigned long *out_rtt) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_FOLD, + rd, ipa, level + }; - arm_smccc_1_1_invoke(SMC_RMI_RTT_FOLD, rd, ipa, level, &res); + invoke_rmi_smc(®s, ®s); if (out_rtt) - *out_rtt = res.a1; + *out_rtt = regs.a1; - return res.a0; + return regs.a0; } /** @@ -388,14 +474,17 @@ static inline int rmi_rtt_fold(unsigned long rd, unsigned long ipa, static inline int rmi_rtt_init_ripas(unsigned long rd, unsigned long base, unsigned long top, unsigned long *out_top) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_INIT_RIPAS, + rd, base, top + }; - arm_smccc_1_1_invoke(SMC_RMI_RTT_INIT_RIPAS, rd, base, top, &res); + invoke_rmi_smc(®s, ®s); if (out_top) - *out_top = res.a1; + *out_top = regs.a1; - return res.a0; + return regs.a0; } /** @@ -414,12 +503,14 @@ static inline int rmi_rtt_map_unprotected(unsigned long rd, long level, unsigned long desc) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_MAP_UNPROTECTED, + rd, ipa, level, desc + }; - arm_smccc_1_1_invoke(SMC_RMI_RTT_MAP_UNPROTECTED, rd, ipa, level, - desc, &res); + invoke_rmi_smc(®s, ®s); - return res.a0; + return regs.a0; } /** @@ -441,7 +532,7 @@ static inline int rmi_rtt_read_entry(unsigned long rd, unsigned long ipa, rd, ipa, level }; - arm_smccc_1_2_smc(®s, ®s); + invoke_rmi_smc(®s, ®s); rtt->walk_level = regs.a1; rtt->state = regs.a2 & 0xFF; @@ -468,14 +559,17 @@ static inline int rmi_rtt_set_ripas(unsigned long rd, unsigned long rec, unsigned long base, unsigned long top, unsigned long *out_top) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_SET_RIPAS, + rd, rec, base, top + }; - arm_smccc_1_1_invoke(SMC_RMI_RTT_SET_RIPAS, rd, rec, base, top, &res); + invoke_rmi_smc(®s, ®s); if (out_top) - *out_top = res.a1; + *out_top = regs.a1; - return res.a0; + return regs.a0; } /** @@ -494,15 +588,400 @@ static inline int rmi_rtt_unmap_unprotected(unsigned long rd, long level, unsigned long *out_top) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RMI_RTT_UNMAP_UNPROTECTED, + rd, ipa, level + }; - arm_smccc_1_1_invoke(SMC_RMI_RTT_UNMAP_UNPROTECTED, rd, ipa, - level, &res); + invoke_rmi_smc(®s, ®s); if (out_top) - *out_top = res.a1; + *out_top = regs.a1; + + return regs.a0; +} + +static inline int rmi_smmu_map(unsigned long pgd, unsigned long iova, + unsigned long pa, unsigned long prot, + unsigned long page_attr, unsigned long *map_cnt) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_MAP, + pgd, iova, pa, prot, page_attr + }; + + invoke_rmi_smc(®s, ®s); + + *map_cnt = regs.a1; + + return regs.a0; +} + +static inline int rmi_smmu_page_table_create(unsigned long pgd, + unsigned long iova, + unsigned long tbl_entry, + unsigned long tbl_size, + unsigned long lvl) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_PAGE_TABLE_CREATE, + pgd, iova, tbl_entry, tbl_size, lvl + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} - return res.a0; +static inline int rmi_smmu_page_table_destroy(unsigned long pgd, + unsigned long iova, + unsigned long tbl_pa, + unsigned long tbl_size, + unsigned long lvl) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_PAGE_TABLE_DESTROY, + pgd, iova, tbl_pa, tbl_size, lvl + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_smmu_ste_write(unsigned long strtab_pa, + unsigned long sid, + unsigned long ste_pa, + unsigned long lvl_strtab) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_STE_WRITE, + strtab_pa, sid, ste_pa, lvl_strtab + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_smmu_config(unsigned long config, unsigned long ns_params) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_CONFIG, + config, ns_params + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_smmu_read_pte(unsigned long pgd, unsigned long iova, + unsigned long lvl, unsigned long *pte) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_READ_PTE, + pgd, iova, lvl + }; + + invoke_rmi_smc(®s, ®s); + + *pte = regs.a1; + return regs.a0; +} + +static inline int rmi_smmu_send_cmdlist(unsigned long ioaddr, + unsigned long rcmds, + unsigned long n, + unsigned long sync) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_SEND_CMDLIST, + ioaddr, rcmds, n, sync + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_smmu_iova_to_phys(unsigned long pgd, unsigned long iova, + unsigned long *phys) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_IOVA_TO_PHYS, + pgd, iova + }; + + invoke_rmi_smc(®s, ®s); + *phys = regs.a1; + + return regs.a0; +} + +static inline int rmi_smmu_read_event(unsigned long ioaddr, u64 *evt) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_READ_EVENT, + ioaddr + }; + + invoke_rmi_smc(®s, ®s); + + evt[0] = regs.a1; + evt[1] = regs.a2; + evt[2] = regs.a3; + evt[3] = regs.a4; + + return regs.a0; +} + +#define MMIO_REG_READ 0 +#define MMIO_REG_WRITE 1 + +#define SMMU_R_REG_32_BIT 32 +#define SMMU_R_REG_64_BIT 64 + +static inline int rmi_smmu_reg_read(unsigned long ioaddr, unsigned long reg, + unsigned long bits, unsigned long *value) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_REG_RW, + ioaddr, + MMIO_REG_READ, + reg, + 0, + bits, + }; + + invoke_rmi_smc(®s, ®s); + + if (regs.a0 == 0) + *value = regs.a1; + + return regs.a0; +} + +static inline int rmi_smmu_reg_read32(unsigned long ioaddr, unsigned long reg, + u32 *value) +{ + unsigned long reg_value; + int ret; + + ret = rmi_smmu_reg_read(ioaddr, reg, SMMU_R_REG_32_BIT, ®_value); + *value = (u32)reg_value; + + return ret; +} + +static inline int rmi_smmu_reg_write(unsigned long ioaddr, unsigned long reg, + unsigned long bits, unsigned long value) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_SMMU_REG_RW, + ioaddr, + MMIO_REG_WRITE, + reg, + value, + bits, + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_smmu_reg_write32(unsigned long ioaddr, unsigned long reg, + u32 value) +{ + return rmi_smmu_reg_write(ioaddr, reg, SMMU_R_REG_32_BIT, + (unsigned long)value); +} + +static inline int rmi_smmu_reg_write64(unsigned long ioaddr, unsigned long reg, + unsigned long value) +{ + return rmi_smmu_reg_write(ioaddr, reg, SMMU_R_REG_64_BIT, value); +} + +#ifdef CONFIG_HISI_CCA_PRE_RESEARCH +#define NEXT_SMMU_CD_WRITE 0x61 +#define NEXT_SMMU_SMMU_CONFIG 0x62 +#define NEXT_SMMU_READ_IPA 0x63 + +static inline int rmi_smmu_cd_write(unsigned long cdtab_pa, + unsigned long ssid, + unsigned long cd_pa, + unsigned long lvl_cdtab) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, NEXT_SMMU_CD_WRITE, + cdtab_pa, ssid, cd_pa, lvl_cdtab + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_next_smmu_config(unsigned long config, + unsigned long ns_params) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, NEXT_SMMU_SMMU_CONFIG, + config, ns_params + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_smmu_read_ipa(unsigned long rd, unsigned long ipa, + unsigned long size, unsigned long buf_pa) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, NEXT_SMMU_READ_IPA, + rd, ipa, size, buf_pa + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} +#endif + +/** + * rmi_granule_io_delegate() - Delegate a Granule + * @phys: PA of the target Granule + * @flags: Flags + * + * Delegate a IO Granule for use by the Realm World. + * + * Return: RMI return code + */ +static inline int rmi_granule_io_delegate(unsigned long phys, unsigned long flags) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_GRANULE_DEV_DELEGATE, + phys, flags + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +/** + * rmi_granule_io_undelegate() - Undelegate a Granule + * @phys: PA of the Granule + * + * Undelegate a Granule to allow use by the Normal World. Will fail if the + * Granule is in use. + * + * Return: RMI return code + */ +static inline int rmi_granule_io_undelegate(unsigned long phys) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_GRANULE_DEV_UNDELEGATE, + phys + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +/** + * rmi_io_create() - Create a IO Granule + * @rd: PA of the RD + * @ipa: IPA at which the granule will be mapped in the guest + * @flags: PA of the source granule + * @rtt: Output structure describing the RTTE + * + * Create an IO Granule. + * + * Return: RMI return code + */ +static inline int rmi_io_create(unsigned long rd, unsigned long pa, + unsigned long ipa) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_DEV_MAP, + rd, pa, ipa + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +/** + * rmi_io_destroy() - Destroy a IO Granule + * @rd: PA of the RD + * @ipa: IPA at which the granule is mapped in the guest + * @data_out: PA of the granule which was destroyed + * @top_out: Top IPA of non-live RTT entries + * + * Transitions the granule to empty, which can be used later. + * + * Return: RMI return code + */ + +static inline int rmi_io_destroy(unsigned long rd, unsigned long ipa, + unsigned long *data_out, + unsigned long *top_out) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_DEV_UNMAP, + rd, ipa + }; + + invoke_rmi_smc(®s, ®s); + + *data_out = regs.a1; + *top_out = regs.a2; + + return regs.a0; +} + + +static inline int rmi_dev_mmio_read(unsigned long reg, unsigned long bits, + unsigned long *value, unsigned long dev_bdf) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_MMIO_RW, + reg, + MMIO_REG_READ, + 0, + bits, + dev_bdf, + }; + + invoke_rmi_smc(®s, ®s); + + if (regs.a0 == 0) + *value = regs.a1; + + return regs.a0; +} + +static inline int rmi_dev_mmio_write(unsigned long reg, unsigned long bits, + unsigned long value, unsigned long dev_bdf) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_MMIO_RW, + reg, + MMIO_REG_WRITE, + value, + bits, + dev_bdf, + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; } static inline int rmi_ccal_delegate_range(unsigned long start_addr, @@ -642,4 +1121,149 @@ static inline int rmi_ccal_data_destroy(unsigned long rd, unsigned long ipa, return regs.a0; } +static inline int rmi_dfx_read_log(unsigned long addr, unsigned long size, + unsigned long mode, + unsigned long *out_len) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCA_DFX_READ_LOG, + addr, size, mode + }; + + arm_smccc_1_2_smc(®s, ®s); + + if (out_len) + *out_len = regs.a1; + + return regs.a0; +} + +static inline int rmi_dfx_set_log_level(unsigned long level) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCA_DFX_SET_LOG_LEVEL, level + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_dfx_set_log_mode(unsigned long mode) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCA_DFX_SET_LOG_MODE, mode + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_dev_init(unsigned long dev_bdf, unsigned long pa) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_INIT, + dev_bdf, pa + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_is_dev_assigned_realm(unsigned long root_bdf, + bool *is_assigned_realm) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_IS_DEV_ASSIGNED, + root_bdf + }; + + invoke_rmi_smc(®s, ®s); + + if (regs.a0 == 0) + *is_assigned_realm = regs.a1; + return regs.a0; +} + +static inline int rmi_dev_assign(unsigned long root_bdf, + unsigned long dev_bdf, + unsigned long assign_to_realm, + unsigned long params_addr, + unsigned long *out_dev_bdf) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_ASSIGN, + root_bdf, dev_bdf, assign_to_realm, params_addr + }; + + invoke_rmi_smc(®s, ®s); + + if (regs.a0 == RMI_ERROR_DEV_INFO) + *out_dev_bdf = regs.a1; + + return regs.a0; +} + +static inline int rmi_dev_unassign(unsigned long root_bdf, unsigned long params_addr) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_UNASSIGN, + root_bdf, params_addr + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_dev_attach(unsigned long dev_bdf, unsigned long rd) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_ATTACH, + dev_bdf, rd + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_dev_detach(unsigned long dev_bdf) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_DETACH, + dev_bdf + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_dfx_perf_monitor(unsigned long addr) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCA_DFX_PERF_MONITOR, addr + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + +static inline int rmi_dev_validate(unsigned long rd_phys, unsigned long rec_phys, + unsigned long dev_bdf, unsigned long guest_dev_bdf) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RMI_HISI_EXT, CCADA_DEV_VALIDATE, + rd_phys, rec_phys, dev_bdf, guest_dev_bdf + }; + + invoke_rmi_smc(®s, ®s); + + return regs.a0; +} + #endif /* __ASM_RMI_CMDS_H */ diff --git a/arch/arm64/include/asm/rmi_smc.h b/arch/arm64/include/asm/rmi_smc.h index c125a2b79cef..76c162e8e02a 100644 --- a/arch/arm64/include/asm/rmi_smc.h +++ b/arch/arm64/include/asm/rmi_smc.h @@ -45,6 +45,14 @@ #define SMC_RMI_RTT_INIT_RIPAS SMC_RMI_CALL(0x0168) #define SMC_RMI_RTT_SET_RIPAS SMC_RMI_CALL(0x0169) +#define SMC_RMI_GRANULE_DEV_DELEGATE SMC_RMI_CALL(0x0170) +#define SMC_RMI_GRANULE_DEV_UNDELEGATE SMC_RMI_CALL(0x0171) +#define SMC_RMI_DEV_MAP SMC_RMI_CALL(0x0172) +#define SMC_RMI_DEV_UNMAP SMC_RMI_CALL(0x0173) +// #define SMC_RMI_MMIO_RW SMC_RMI_CALL(0x017C) +// #define SMC_RMI_MMIO_REG_RW SMC_RMI_CALL(0x017D) +// #define SMC_RMI_DEV_DELEGATE SMC_RMI_CALL(0x017E) + #define SMC_RMI_HISI_EXT SMC_RMI_CALL(0x018F) enum hisi_ext_cmd { @@ -58,6 +66,30 @@ enum hisi_ext_cmd { CCAL_BLOCK_DATA_CREATE_LVL2, CCAL_BLOCK_DATA_CREATE_UNKNOWN_LVL2, CCAL_DATA_DESTROY, + + CCADA_SMMU_MAP = 0x20, + CCADA_SMMU_PAGE_TABLE_CREATE, + CCADA_SMMU_PAGE_TABLE_DESTROY, + CCADA_SMMU_STE_WRITE, + CCADA_SMMU_CONFIG, + CCADA_SMMU_REG_RW, + CCADA_SMMU_READ_PTE, + CCADA_SMMU_SEND_CMDLIST, + CCADA_SMMU_IOVA_TO_PHYS, + CCADA_SMMU_READ_EVENT, + CCADA_DEV_INIT, + CCADA_IS_DEV_ASSIGNED, + CCADA_DEV_ASSIGN, + CCADA_DEV_UNASSIGN, + CCADA_DEV_ATTACH, + CCADA_DEV_DETACH, + CCADA_DEV_VALIDATE, + CCADA_DEV_MMIO_RW, + + CCA_DFX_PERF_MONITOR = 0x40, + CCA_DFX_SET_LOG_LEVEL, + CCA_DFX_SET_LOG_MODE, + CCA_DFX_READ_LOG, }; #define RMI_ABI_MAJOR_VERSION 1 @@ -70,6 +102,7 @@ enum hisi_ext_cmd { #define RMI_UNASSIGNED 0 #define RMI_ASSIGNED 1 #define RMI_TABLE 2 +#define RMI_ASSIGNED_IO_PRIVATE 3 #define RMI_RETURN_STATUS(ret) ((ret) & 0xFF) #define RMI_RETURN_INDEX(ret) (((ret) >> 8) & 0xFF) @@ -79,11 +112,13 @@ enum hisi_ext_cmd { #define RMI_ERROR_REALM 2 #define RMI_ERROR_REC 3 #define RMI_ERROR_RTT 4 +#define RMI_ERROR_DEV_INFO 9 enum rmi_ripas { RMI_EMPTY = 0, RMI_RAM = 1, RMI_DESTROYED = 2, + RMI_IO = 3, }; #define RMI_NO_MEASURE_CONTENT 0 @@ -201,7 +236,11 @@ struct rec_enter { }; u8 padding2[0x100]; }; - u8 padding3[0x400]; + union { /* 0x400 */ + u64 clidr_el1; + u8 padding3[0x100]; + }; + u8 padding4[0x300]; }; #define RMI_EXIT_SYNC 0x00 @@ -211,6 +250,7 @@ struct rec_enter { #define RMI_EXIT_RIPAS_CHANGE 0x04 #define RMI_EXIT_HOST_CALL 0x05 #define RMI_EXIT_SERROR 0x06 +#define RMI_EXIT_VALIDATE_DEV 0xb struct rec_exit { union { /* 0x000 */ @@ -263,7 +303,15 @@ struct rec_exit { struct { u8 pmu_ovf_status; }; - u8 padding7[0x100]; + u8 padding7[0x80]; + }; + union { /* 0x780 */ + struct { + u16 guest_dev_bdf; + u16 dev_bdf; + bool valid; + }; + u8 padding8[0x20]; }; }; diff --git a/arch/arm64/include/asm/rsi.h b/arch/arm64/include/asm/rsi.h index 5f9c8623183d..6610c5d3076a 100644 --- a/arch/arm64/include/asm/rsi.h +++ b/arch/arm64/include/asm/rsi.h @@ -65,4 +65,26 @@ static inline int rsi_set_memory_range_shared(phys_addr_t start, return rsi_set_memory_range(start, end, RSI_RIPAS_EMPTY, RSI_CHANGE_DESTROYED); } +#ifdef CONFIG_HISI_CCA_DA +/* + * Convert the specified range to DEV. + */ +static inline int rsi_set_mmio_range_protected(phys_addr_t addr, size_t len) +{ + return rsi_set_memory_range(addr, addr + len, RSI_RIPAS_DEV, + RSI_CHANGE_DESTROYED); +} + +static inline bool rsi_is_realm_dev(u16 bdf) +{ + bool is_realm_dev = false; + unsigned long ret; + + ret = rsi_validate_dev(bdf, &is_realm_dev); + WARN_ON(ret); + + return is_realm_dev; +} +#endif /* CONFIG_HISI_CCA_DA */ + #endif /* __ASM_RSI_H_ */ diff --git a/arch/arm64/include/asm/rsi_cmds.h b/arch/arm64/include/asm/rsi_cmds.h index e6a211001bd3..ace1908fe030 100644 --- a/arch/arm64/include/asm/rsi_cmds.h +++ b/arch/arm64/include/asm/rsi_cmds.h @@ -9,6 +9,7 @@ #include #include +#include #define RSI_GRANULE_SHIFT 12 #define RSI_GRANULE_SIZE (_AC(1, UL) << RSI_GRANULE_SHIFT) @@ -24,25 +25,31 @@ static inline unsigned long rsi_request_version(unsigned long req, unsigned long *out_lower, unsigned long *out_higher) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RSI_ABI_VERSION, + req + }; - arm_smccc_smc(SMC_RSI_ABI_VERSION, req, 0, 0, 0, 0, 0, 0, &res); + invoke_rsi_smc(®s, ®s); if (out_lower) - *out_lower = res.a1; + *out_lower = regs.a1; if (out_higher) - *out_higher = res.a2; + *out_higher = regs.a2; - return res.a0; + return regs.a0; } static inline unsigned long rsi_get_realm_config(struct realm_config *cfg) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RSI_REALM_CONFIG, + virt_to_phys(cfg) + }; - arm_smccc_smc(SMC_RSI_REALM_CONFIG, virt_to_phys(cfg), - 0, 0, 0, 0, 0, 0, &res); - return res.a0; + invoke_rsi_smc(®s, ®s); + + return regs.a0; } static inline unsigned long rsi_ipa_state_get(phys_addr_t start, @@ -50,20 +57,21 @@ static inline unsigned long rsi_ipa_state_get(phys_addr_t start, enum ripas *state, phys_addr_t *top) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RSI_IPA_STATE_GET, + start, end + }; - arm_smccc_smc(SMC_RSI_IPA_STATE_GET, - start, end, 0, 0, 0, 0, 0, - &res); + invoke_rsi_smc(®s, ®s); - if (res.a0 == RSI_SUCCESS) { + if (regs.a0 == RSI_SUCCESS) { if (top) - *top = res.a1; + *top = regs.a1; if (state) - *state = res.a2; + *state = regs.a2; } - return res.a0; + return regs.a0; } static inline long rsi_set_addr_range_state(phys_addr_t start, @@ -72,18 +80,20 @@ static inline long rsi_set_addr_range_state(phys_addr_t start, unsigned long flags, phys_addr_t *top) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RSI_IPA_STATE_SET, + start, end, state, flags + }; - arm_smccc_smc(SMC_RSI_IPA_STATE_SET, start, end, state, - flags, 0, 0, 0, &res); + invoke_rsi_smc(®s, ®s); if (top) - *top = res.a1; + *top = regs.a1; - if (res.a2 != RSI_ACCEPT) + if (regs.a2 != RSI_ACCEPT) return -EPERM; - return res.a0; + return regs.a0; } /** @@ -115,7 +125,7 @@ rsi_attestation_token_init(const u8 *challenge, unsigned long size) regs.a0 = SMC_RSI_ATTESTATION_TOKEN_INIT; memcpy(®s.a1, challenge, size); - arm_smccc_1_2_smc(®s, ®s); + invoke_rsi_smc(®s, ®s); if (regs.a0 == RSI_SUCCESS) return regs.a1; @@ -147,14 +157,36 @@ static inline unsigned long rsi_attestation_token_continue(phys_addr_t granule, unsigned long size, unsigned long *len) { - struct arm_smccc_res res; + struct arm_smccc_1_2_regs regs = { + SMC_RSI_ATTESTATION_TOKEN_CONTINUE, + granule, offset, size + }; - arm_smccc_1_1_invoke(SMC_RSI_ATTESTATION_TOKEN_CONTINUE, - granule, offset, size, 0, &res); + invoke_rsi_smc(®s, ®s); if (len) - *len = res.a1; - return res.a0; + *len = regs.a1; + return regs.a0; +} + +/** + * rsi_validate_dev - Validate if a pci dev is deleagted to realm or not. + * + * @bdf: The bdf of the pci dev. + * @is_realm_dev: True if the dev is delegated to realm. + */ +static inline unsigned long rsi_validate_dev(unsigned long bdf, bool *is_realm_dev) +{ + struct arm_smccc_1_2_regs regs = { + SMC_RSI_VALIDATE_DEV, + bdf + }; + + invoke_rsi_smc(®s, ®s); + + if (is_realm_dev) + *is_realm_dev = !!regs.a1; + return regs.a0; } #endif /* __ASM_RSI_CMDS_H */ diff --git a/arch/arm64/include/asm/rsi_smc.h b/arch/arm64/include/asm/rsi_smc.h index 6cb070eca9e9..a5a582606ad3 100644 --- a/arch/arm64/include/asm/rsi_smc.h +++ b/arch/arm64/include/asm/rsi_smc.h @@ -190,4 +190,12 @@ struct realm_config { */ #define SMC_RSI_HOST_CALL SMC_RSI_FID(0x199) +/* + * arg1 == bdf of pci device + * ret0 == Status / error + * ret1 == True if it device is realm / false + */ +#define SMC_RSI_VALIDATE_DEV SMC_RSI_FID(0x19A) + + #endif /* __ASM_RSI_SMC_H_ */ diff --git a/arch/arm64/include/uapi/asm/kvm.h b/arch/arm64/include/uapi/asm/kvm.h index 04215d303762..0b12147ef86c 100644 --- a/arch/arm64/include/uapi/asm/kvm.h +++ b/arch/arm64/include/uapi/asm/kvm.h @@ -208,6 +208,12 @@ struct kvm_arm_counter_offset { __u64 reserved; }; +struct kvm_arm_read_ipa { + __u64 guest_ipa; + __u64 length; + void __user *addr; +}; + #define KVM_ARM_TAGS_TO_GUEST 0 #define KVM_ARM_TAGS_FROM_GUEST 1 @@ -430,7 +436,8 @@ enum { #define KVM_CAP_ARM_RME_INIT_RIPAS_REALM 2 #define KVM_CAP_ARM_RME_POPULATE_REALM 3 #define KVM_CAP_ARM_RME_ACTIVATE_REALM 4 -#define KVM_CAP_ARM_RME_MAP_RAM_CCAL 5 +#define KVM_CAP_ARM_RME_MAP_RAM_REALM 5 +#define KVM_CAP_ARM_RME_MAP_RAM_CCAL 6 /* List of configuration items accepted for KVM_CAP_ARM_RME_CONFIG_REALM */ #define ARM_RME_CONFIG_RPV 0 @@ -474,6 +481,12 @@ struct arm_rme_init_ripas { __u64 reserved[2]; }; +struct arm_rme_map_ram_args { + __u64 ram_base; + __u64 ram_size; + __u32 reserved[4]; +}; + /* Device Control API on vcpu fd */ #define KVM_ARM_VCPU_PMU_V3_CTRL 0 #define KVM_ARM_VCPU_PMU_V3_IRQ 0 diff --git a/arch/arm64/kernel/Makefile b/arch/arm64/kernel/Makefile index d67126570cb2..300bfcb8a890 100644 --- a/arch/arm64/kernel/Makefile +++ b/arch/arm64/kernel/Makefile @@ -34,7 +34,7 @@ obj-y := debug-monitors.o entry.o irq.o fpsimd.o \ cpufeature.o alternative.o cacheinfo.o \ smp.o smp_spin_table.o topology.o smccc-call.o \ syscall.o proton-pack.o idreg-override.o idle.o \ - patching.o rsi.o + patching.o rsi.o realm_guest.o obj-$(CONFIG_AARCH32_EL0) += binfmt_elf32.o sys32.o signal32.o \ sys_compat.o diff --git a/arch/arm64/kernel/realm_guest.c b/arch/arm64/kernel/realm_guest.c new file mode 100644 index 000000000000..8daba08c41e8 --- /dev/null +++ b/arch/arm64/kernel/realm_guest.c @@ -0,0 +1,50 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* + * Copyright (C) 2025. Huawei Technologies Co., Ltd. All rights reserved. + */ +#include +#include + +static struct device realm_alloc_device; + +/* + * struct io_tlb_no_swiotlb_mem - whether use the + * bounce buffer mechanism or not + * @for_alloc: %true if the pool is used for memory allocation. + * Here it is set to %false, to force devices to use direct dma operations. + * + * @force_bounce: %true if swiotlb bouncing is forced. + * Here it is set to %false, to force devices to use direct dma operations. + */ +static struct io_tlb_mem io_tlb_no_swiotlb_mem = { + .for_alloc = false, + .force_bounce = false, +}; + +void enable_swiotlb_for_realm_dev(struct device *dev, bool enable) +{ + if (!is_realm_world()) + return; + + if (enable) + swiotlb_dev_init(dev); + else + dev->dma_io_tlb_mem = &io_tlb_no_swiotlb_mem; +} +EXPORT_SYMBOL_GPL(enable_swiotlb_for_realm_dev); + +void __init realm_guest_init(void) +{ + device_initialize(&realm_alloc_device); + enable_swiotlb_for_realm_dev(&realm_alloc_device, true); +} + +struct page *realm_alloc_shared_pages(gfp_t gfp, unsigned int order) +{ + return swiotlb_alloc(&realm_alloc_device, (1 << order) * PAGE_SIZE); +} + +void realm_free_shared_pages(void *addr, unsigned int order) +{ + swiotlb_free(&realm_alloc_device, (struct page *)addr, (1 << order) * PAGE_SIZE); +} diff --git a/arch/arm64/kernel/rsi.c b/arch/arm64/kernel/rsi.c index be868cb5a4b4..66514a08ea5d 100644 --- a/arch/arm64/kernel/rsi.c +++ b/arch/arm64/kernel/rsi.c @@ -12,6 +12,7 @@ #include #include +#include #include static struct realm_config config; @@ -140,6 +141,9 @@ static int realm_ioremap_hook(phys_addr_t phys, size_t size, pgprot_t *prot) void __init arm64_rsi_init(void) { +#ifdef CONFIG_CCA_RME_PMU + RSI_PERF_INIT(); +#endif if (arm_smccc_1_1_get_conduit() != SMCCC_CONDUIT_SMC) return; if (!rsi_version_matches()) @@ -160,6 +164,8 @@ void __init arm64_rsi_init(void) arm64_rsi_setup_memory(); + realm_guest_init(); + static_branch_enable(&rsi_present); } diff --git a/arch/arm64/kvm/Kconfig b/arch/arm64/kvm/Kconfig index 52edbd7f6340..1073aa76604d 100644 --- a/arch/arm64/kvm/Kconfig +++ b/arch/arm64/kvm/Kconfig @@ -100,3 +100,11 @@ config VIRT_VTIMER_IRQ_BYPASS default n endif # VIRTUALIZATION + +config CCA_RME_PMU + bool "Enable RME performance monitor" + default n + help + Say Y here to enable RME performance monitor for RMI and RSI calls + + If unsure, say N. diff --git a/arch/arm64/kvm/Makefile b/arch/arm64/kvm/Makefile index 8542c4be71ab..8b2bd645c1eb 100644 --- a/arch/arm64/kvm/Makefile +++ b/arch/arm64/kvm/Makefile @@ -28,6 +28,8 @@ kvm-$(CONFIG_HW_PERF_EVENTS) += pmu-emul.o pmu.o kvm-$(CONFIG_HISI_VIRTCCA_HOST) += tmi.o kvm-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm.o kvm-$(CONFIG_HISI_VIRTCCA_HOST) += virtcca_cvm_exit.o +kvm-$(CONFIG_HISI_CCA_DA) += rme-da.o +kvm-$(CONFIG_CCA_RME_PMU) += rme-pmu.o obj-$(CONFIG_KVM_HISI_VIRT) += hisilicon/ always-y := hyp_constants.h hyp-constants.s diff --git a/arch/arm64/kvm/arm.c b/arch/arm64/kvm/arm.c index d7511196e607..9dbca0d2ade6 100644 --- a/arch/arm64/kvm/arm.c +++ b/arch/arm64/kvm/arm.c @@ -46,6 +46,9 @@ #include #include #include +#ifdef CONFIG_HISI_CCA_DA +#include +#endif static enum kvm_mode kvm_mode = KVM_MODE_DEFAULT; @@ -1945,6 +1948,17 @@ long kvm_arch_vcpu_ioctl(struct file *filp, return -EFAULT; return kvm_arm_vcpu_rmm_psci_complete(vcpu, &req); } +#ifdef CONFIG_HISI_CCA_DA + case KVM_ARM_VCPU_RME_DEV_VALIDATE: { + struct kvm_arm_rme_dev_validate args; + + if (!_vcpu_is_rec(vcpu)) + return -EPERM; + if (copy_from_user(&args, argp, sizeof(args))) + return -EFAULT; + return kvm_arm_vcpu_rme_dev_validate(vcpu, &args); + } +#endif default: r = -EINVAL; } diff --git a/arch/arm64/kvm/mmu.c b/arch/arm64/kvm/mmu.c index 9873838ef342..5f4ba2b2b901 100644 --- a/arch/arm64/kvm/mmu.c +++ b/arch/arm64/kvm/mmu.c @@ -329,7 +329,7 @@ static void __unmap_stage2_range(struct kvm_s2_mmu *mmu, phys_addr_t start, u64 WARN_ON(size & ~PAGE_MASK); if (_kvm_is_realm(kvm)) - kvm_realm_unmap_range(kvm, start, size, !only_shared); + kvm_realm_unmap_range(kvm, start, size, !only_shared, kvm->arch.realm.delay_undelegate); else WARN_ON(stage2_apply_range(mmu, start, end, kvm_pgtable_stage2_unmap, @@ -349,7 +349,7 @@ static void stage2_flush_memslot(struct kvm *kvm, phys_addr_t end = addr + PAGE_SIZE * memslot->npages; if (_kvm_is_realm(kvm)) - kvm_realm_unmap_range(kvm, addr, end - addr, false); + kvm_realm_unmap_range(kvm, addr, end - addr, false, false); else stage2_apply_range_resched(&kvm->arch.mmu, addr, end, kvm_pgtable_stage2_flush); @@ -1455,6 +1455,12 @@ static int realm_map_ipa(struct kvm *kvm, phys_addr_t ipa, return realm_map_non_secure(realm, ipa, pfn, map_size, memcache); +#ifdef CONFIG_HISI_CCA_DA + if (kvm_is_device_pfn(pfn)) + return realm_map_mmio_protected(realm, ipa, pfn, map_size, + memcache); +#endif + return realm_map_protected(realm, ipa, pfn, map_size, memcache); } diff --git a/arch/arm64/kvm/rme-da.c b/arch/arm64/kvm/rme-da.c new file mode 100644 index 000000000000..1d531ab95a57 --- /dev/null +++ b/arch/arm64/kvm/rme-da.c @@ -0,0 +1,794 @@ +// SPDX-License-Identifier: GPL-2.0 +#include +#include +#include +#include + +#include +#include +#include +#include + +#define MAX_REALM_DEV_NUM_ORDER 8 +#define PCI_DEVICE_ID_HUAWEI_ZIP_PF 0xa250 +#define PCI_DEVICE_ID_HUAWEI_SEC_PF 0xa255 +#define PCI_DEVICE_ID_HUAWEI_HPRE_PF 0xa258 +#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) +#define PCI_BDF_MASK 0xffff + +struct realm_pcipc_ns_entry { + struct device *dev; + struct hlist_node node; +}; + +static DEFINE_HASHTABLE(g_pcipc_ns_htable, MAX_REALM_DEV_NUM_ORDER); +static DEFINE_SPINLOCK(g_realm_hash_lock); + +static int realm_pcipc_ns_add_dev(struct device *dev) +{ + struct realm_pcipc_ns_entry *entry; + + if (!dev) + return -EINVAL; + + if (is_realm_pcipc_ns(dev)) + return 0; + + spin_lock(&g_realm_hash_lock); + entry = kzalloc(sizeof(*entry), GFP_ATOMIC); + if (!entry) { + spin_unlock(&g_realm_hash_lock); + return -ENOMEM; + } + entry->dev = dev; + hash_add(g_pcipc_ns_htable, &entry->node, (unsigned long)dev); + spin_unlock(&g_realm_hash_lock); + + return 0; +} + +static void realm_pcipc_ns_remove_dev(struct device *dev) +{ + struct hlist_node *next; + struct realm_pcipc_ns_entry *entry; + + if (!dev) + return; + + spin_lock(&g_realm_hash_lock); + hash_for_each_possible_safe(g_pcipc_ns_htable, entry, next, node, (unsigned long)dev) { + if (entry->dev == dev) { + hash_del(&entry->node); + kfree(entry); + break; + } + } + spin_unlock(&g_realm_hash_lock); +} + +void realm_pcipc_ns_remove(const struct pci_device_id *id_table) +{ + const struct pci_device_id *ent; + struct pci_dev *pdev = NULL; + + for_each_pci_dev(pdev) { + ent = pci_match_id(id_table, pdev); + if (!ent) + continue; + realm_pcipc_ns_remove_dev(&pdev->dev); + } +} +EXPORT_SYMBOL_GPL(realm_pcipc_ns_remove); + +/* + * realm_pcipc_ns_add should be called before pci_register_driver + */ +int realm_pcipc_ns_add(const struct pci_device_id *id_table) +{ + const struct pci_device_id *ent; + struct pci_dev *pdev = NULL; + int ret = 0; + + for_each_pci_dev(pdev) { + ent = pci_match_id(id_table, pdev); + if (!ent) + continue; + + ret = realm_pcipc_ns_add_dev(&pdev->dev); + if (ret) + break; + } + + if (ret) + realm_pcipc_ns_remove(id_table); + + return ret; +} +EXPORT_SYMBOL_GPL(realm_pcipc_ns_add); + +bool is_realm_pcipc_ns(struct device *dev) +{ + struct realm_pcipc_ns_entry *entry; + + if (!dev) + return false; + + spin_lock(&g_realm_hash_lock); + hash_for_each_possible(g_pcipc_ns_htable, entry, node, (unsigned long)dev) { + if (entry->dev == dev) { + spin_unlock(&g_realm_hash_lock); + return true; + } + } + spin_unlock(&g_realm_hash_lock); + + return false; +} + +/** + * get_child_devices_rec - Traverse pcie topology to find child devices + * If dev is a bridge, get it's children + * If dev is a regular device, get itself + * @dev: Device for which to get child devices + * @devs: All child devices under input dev + * @max_devs: Max num of devs + * @ndev: Num of child devices + */ +static void get_child_devices_rec(struct pci_dev *dev, uint16_t *devs, + int max_devs, int *ndev) +{ + struct pci_bus *bus = dev->subordinate; + + if (bus) { /* dev is a bridge */ + struct pci_dev *child; + + list_for_each_entry(child, &bus->devices, bus_list) { + get_child_devices_rec(child, devs, max_devs, ndev); + } + } else { /* dev is a regular device */ + uint16_t bdf = pci_dev_id(dev); + int i; + /* check if bdf is already in devs */ + for (i = 0; i < *ndev; i++) { + if (devs[i] == bdf) + return; + } + /* check overflow */ + if (*ndev >= max_devs) { + pr_warn("devices num over max devs\n"); + return; + } + devs[*ndev] = bdf; + *ndev = *ndev + 1; + } +} + +/** + * rme_get_all_dev_info - Retrieve all devices under the root port + * @rp_dev: root port device + * @params: Assign device parameters + * + * Returns: + * %0 if get all devices under the root port successful + * %-EINVAL if the total number of devices under the root port exceeds the maximum + */ +static int rme_get_all_dev_info(struct pci_dev *rp_dev, + struct rmi_dev_assign_params *params) +{ + int ret = 0; + int ndev = 0; + + params->root_bdf = pci_dev_id(rp_dev); + get_child_devices_rec(rp_dev, params->devs, MAX_DEV_PER_PORT, &ndev); + params->num_dev = ndev; + if (params->num_dev >= MAX_DEV_PER_PORT) { + ret = -EINVAL; + pci_err(rp_dev, "Dev nums overflow\n"); + return ret; + } + return ret; +} + +static bool is_hisi_acc_dev(struct pci_dev *pdev) +{ + switch (pdev->vendor) { + case PCI_VENDOR_ID_HUAWEI: + switch (pdev->device) { + case PCI_DEVICE_ID_HUAWEI_ZIP_PF: + case PCI_DEVICE_ID_HUAWEI_SEC_PF: + case PCI_DEVICE_ID_HUAWEI_HPRE_PF: + case PCI_DEVICE_ID_HUAWEI_ZIP_VF: + case PCI_DEVICE_ID_HUAWEI_SEC_VF: + case PCI_DEVICE_ID_HUAWEI_HPRE_VF: + return true; + default: + return false; + } + default: + return false; + } +} + +static struct pci_dev *rme_get_root_dev(struct pci_dev *pdev) +{ + if (is_hisi_acc_dev(pdev)) + return pci_physfn(pdev); + + return pcie_find_root_port(pdev); +} + +bool is_support_rme(void) +{ + return static_branch_unlikely(&kvm_rme_is_available); +} +EXPORT_SYMBOL_GPL(is_support_rme); + +static inline bool is_dev_assigned_realm(struct pci_dev *pdev) +{ + struct pci_dev *root_dev; + bool is_assigned_realm; + + root_dev = rme_get_root_dev(pdev); + if (!root_dev) + return false; + + if (rmi_is_dev_assigned_realm(pci_dev_id(root_dev), &is_assigned_realm)) + return false; + + return is_assigned_realm; +} + +#define MMIO_REG_32_BIT 32 +static u32 rme_mmio_read32(unsigned long addr, struct pci_dev *pdev) +{ + unsigned long value; + int ret; + + ret = rmi_dev_mmio_read(addr, MMIO_REG_32_BIT, &value, pci_dev_id(pdev)); + if (ret) { + pr_err("rmi_dev_mmio_read error, ret = %d\n", ret); + return 0; + } + + return value; +} + +static void rme_mmio_write32(unsigned long addr, unsigned long value, + struct pci_dev *pdev) +{ + int ret; + + ret = rmi_dev_mmio_write(addr, MMIO_REG_32_BIT, value, pci_dev_id(pdev)); + if (ret) + pr_err("rmi_dev_mmio_write error, ret = %d\n", ret); +} + +static inline u64 get_pci_desc_pbase(struct pci_dev *dev, u16 msi_index) +{ + u64 table_pbase, desc_pbase; + u32 table_offset; + u8 bir; + + /* Get the MSI-X table physical address */ + pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE, &table_offset); + bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR); + table_offset &= PCI_MSIX_TABLE_OFFSET; + table_pbase = pci_resource_start(dev, bir) + table_offset; + + /* Get the MSI-X entry physical address */ + desc_pbase = table_pbase + msi_index * PCI_MSIX_ENTRY_SIZE; + + return desc_pbase; +} + +bool rme_dev_pci_read_msi_msg(struct msi_desc *desc, struct msi_msg *msg) +{ + struct pci_dev *dev = msi_desc_to_pci_dev(desc); + u64 pbase; + + if (!is_support_rme() || !is_dev_assigned_realm(dev)) + return false; + + pbase = get_pci_desc_pbase(dev, desc->msi_index); + + msg->address_lo = rme_mmio_read32(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, dev); + msg->address_hi = rme_mmio_read32(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, dev); + msg->data = rme_mmio_read32(pbase + PCI_MSIX_ENTRY_DATA, dev); + + return true; +} +EXPORT_SYMBOL_GPL(rme_dev_pci_read_msi_msg); + +bool rme_dev_pci_write_msg_msi(struct msi_desc *desc, struct msi_msg *msg) +{ + struct pci_dev *dev = msi_desc_to_pci_dev(desc); + u64 pbase; + u32 ctrl; + bool unmasked; + + if (!is_support_rme() || !is_dev_assigned_realm(dev)) + return false; + + pbase = get_pci_desc_pbase(dev, desc->msi_index); + + ctrl = desc->pci.msix_ctrl; + unmasked = !(ctrl & PCI_MSIX_ENTRY_CTRL_MASKBIT); + + rme_mmio_write32(pbase + PCI_MSIX_ENTRY_LOWER_ADDR, msg->address_lo, dev); + rme_mmio_write32(pbase + PCI_MSIX_ENTRY_UPPER_ADDR, msg->address_hi, dev); + rme_mmio_write32(pbase + PCI_MSIX_ENTRY_DATA, msg->data, dev); + + if (unmasked && desc->pci.msi_attrib.can_mask) + rme_mmio_write32(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, ctrl, dev); + + /* Ensure that the writes are visible in the device */ + rme_mmio_read32(pbase + PCI_MSIX_ENTRY_DATA, dev); + + return true; +} +EXPORT_SYMBOL_GPL(rme_dev_pci_write_msg_msi); + +bool rme_dev_msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) +{ + u64 pbase; + + if (!is_support_rme() || !is_dev_assigned_realm(dev)) + return false; + + pbase = get_pci_desc_pbase(dev, desc->msi_index); + + desc->pci.msix_ctrl = rme_mmio_read32(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, dev); + + return true; +} +EXPORT_SYMBOL_GPL(rme_dev_msix_prepare_msi_desc); + +bool rme_dev_pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) +{ + struct pci_dev *dev = msi_desc_to_pci_dev(desc); + u64 pbase; + + if (!is_support_rme() || !is_dev_assigned_realm(dev)) + return false; + + pbase = get_pci_desc_pbase(dev, desc->msi_index); + + if (desc->pci.msi_attrib.can_mask) + rme_mmio_write32(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, ctrl, dev); + + return true; +} +EXPORT_SYMBOL_GPL(rme_dev_pci_msix_write_vector_ctrl); + +bool rme_dev_pci_msix_mask(struct msi_desc *desc) +{ + struct pci_dev *dev = msi_desc_to_pci_dev(desc); + u64 pbase; + + if (!is_support_rme() || !is_dev_assigned_realm(dev)) + return false; + + pbase = get_pci_desc_pbase(dev, desc->msi_index); + + /* Flush write to device */ + rme_mmio_read32(pbase, dev); + + return true; +} +EXPORT_SYMBOL_GPL(rme_dev_pci_msix_mask); + +bool rme_dev_msix_mask_all(struct pci_dev *dev, int tsize) +{ + u64 pbase; + u32 ctrl = PCI_MSIX_ENTRY_CTRL_MASKBIT; + u16 rw_ctrl; + int i; + + if (!is_support_rme() || !is_dev_assigned_realm(dev)) + return false; + + pbase = get_pci_desc_pbase(dev, 0); + + if (pci_msi_ignore_mask) + goto out; + + for (i = 0; i < tsize; i++, pbase += PCI_MSIX_ENTRY_SIZE) + rme_mmio_write32(pbase + PCI_MSIX_ENTRY_VECTOR_CTRL, ctrl, dev); + +out: + pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &rw_ctrl); + rw_ctrl &= ~PCI_MSIX_FLAGS_MASKALL; + rw_ctrl |= 0; + pci_write_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, rw_ctrl); + + pcibios_free_irq(dev); + return true; +} +EXPORT_SYMBOL_GPL(rme_dev_msix_mask_all); + +static int set_msix_region_shared(struct pci_dev *pdev, int bar) +{ + resource_size_t start; + resource_size_t end; + u32 table_offset; + u16 control; + u8 msix_bir; + u8 tsize; + + pci_read_config_dword(pdev, pdev->msix_cap + PCI_MSIX_TABLE, &table_offset); + msix_bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR); + if (msix_bir != bar) + return 0; + + table_offset &= PCI_MSIX_TABLE_OFFSET; + start = pci_resource_start(pdev, msix_bir) + table_offset; + + pci_read_config_word(pdev, pdev->msix_cap + PCI_MSIX_FLAGS, &control); + tsize = msix_table_size(control); + end = ALIGN(start + tsize * PCI_MSIX_ENTRY_SIZE, PAGE_SIZE); + + return rsi_set_memory_range_shared(start, end); +} + +int ccada_init_mem_region(struct pci_dev *pdev, int bar) +{ + int ret; + + if (!is_realm_world()) + return 0; + + if (!pdev || bar < 0 || bar >= PCI_STD_NUM_BARS) + return -EINVAL; + + if (pci_resource_len(pdev, bar) == 0) + return 0; + + if (!rsi_is_realm_dev(pci_dev_id(pdev))) + return 0; + + ret = rsi_set_mmio_range_protected(pci_resource_start(pdev, bar), + pci_resource_len(pdev, bar)); + if (ret) { + pci_err(pdev, "Failed to set bar[%d] to protected\n", bar); + return ret; + } + + if (pdev->msix_cap) { + ret = set_msix_region_shared(pdev, bar); + if (ret) + pci_err(pdev, "Failed to set msi-x to shared\n"); + return ret; + } + + return 0; +} +EXPORT_SYMBOL_GPL(ccada_init_mem_region); + +/** + * rme_mmio_va_to_pa - To convert the virtual address of the mmio space + * to a physical address, it is necessary to implement this interface + * because the kernel insterface __pa has an error when converting the + * physical address of the virtual address of the mmio space + * @addr: MMIO virtual address + */ +static u64 rme_mmio_va_to_pa(const volatile void *addr) +{ + uint64_t pa, par_el1; + + asm volatile( + "AT S1E1W, %0\n" + ::"r"((uint64_t)(addr)) + ); + isb(); + asm volatile( + "mrs %0, par_el1\n" + : "=r"(par_el1) + ); + + pa = ((uint64_t)(addr) & (PAGE_SIZE - 1)) | + (par_el1 & ULL(0x000ffffffffff000)); + + if (par_el1 & UL(1 << 0)) + return (uint64_t)(addr); + else + return pa; +} + +u32 readl_cca_hook(volatile void __iomem *addr, struct pci_dev *pdev) +{ + if (is_support_rme() && is_dev_assigned_realm(pdev)) + return rme_mmio_read32(rme_mmio_va_to_pa(addr), pdev); + + return readl(addr); +} +EXPORT_SYMBOL_GPL(readl_cca_hook); + +void writel_cca_hook(u32 val, volatile void __iomem *addr, struct pci_dev *pdev) +{ + if (is_support_rme() && is_dev_assigned_realm(pdev)) + rme_mmio_write32(rme_mmio_va_to_pa(addr), val, pdev); + + writel(val, addr); +} +EXPORT_SYMBOL_GPL(writel_cca_hook); + +void __raw_writel_cca_hook(u32 val, volatile void __iomem *addr, + struct pci_dev *pdev) +{ + if (is_support_rme() && is_dev_assigned_realm(pdev)) + rme_mmio_write32(rme_mmio_va_to_pa(addr), val, pdev); + + __raw_writel(val, addr); +} +EXPORT_SYMBOL_GPL(__raw_writel_cca_hook); + +static int rme_dev_assign(u16 root_bdf, u16 dev_bdf, bool assign_to_realm, + phys_addr_t params_addr) +{ + unsigned long out_dev_bdf = ~0; + unsigned long last_dev_bdf = ~0; + phys_addr_t phys; + void *virt; + int ret; + +retry: + ret = rmi_dev_assign(root_bdf, dev_bdf, assign_to_realm, params_addr, + &out_dev_bdf); + if (ret == RMI_ERROR_DEV_INFO) { + if (out_dev_bdf == last_dev_bdf) + return -ENXIO; + + virt = (void *)get_zeroed_page(GFP_KERNEL); + if (!virt) { + pr_err("Failed to allocate page for dev %#lx\n", + out_dev_bdf); + return -ENOMEM; + } + + phys = virt_to_phys(virt); + if (rmi_granule_delegate(phys)) { + free_page((unsigned long)virt); + return -ENXIO; + } + + ret = rmi_dev_init(out_dev_bdf, phys); + if (ret) { + if (WARN_ON(rmi_granule_undelegate(phys))) { + /* leak the page */ + return -ENXIO; + } + free_page((unsigned long)virt); + return -ENXIO; + } + last_dev_bdf = out_dev_bdf; + goto retry; + } else if (WARN_ON(ret)) { + return -ENXIO; + } + + return 0; +} + +int kvm_rme_dev_assign(struct pci_dev *pdev, struct kvm *kvm) +{ + struct rmi_dev_assign_params *params = NULL; + struct pci_dev *root_dev; + int ret; + + if (!is_support_rme() || !pdev || !kvm) + return -EINVAL; + + root_dev = rme_get_root_dev(pdev); + if (!root_dev) { + if (kvm_is_realm(kvm)) + return -EINVAL; + else + return 0; + } + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) + return -ENOMEM; + + ret = rme_get_all_dev_info(root_dev, params); + if (ret) { + kfree(params); + return ret; + } + + ret = rme_dev_assign(pci_dev_id(root_dev), pci_dev_id(pdev), + kvm_is_realm(kvm), virt_to_phys(params)); + if (ret) + pci_err(root_dev, "Failed to assign dev\n"); + else if (!ret && kvm_is_realm(kvm)) + kvm->arch.realm.delay_undelegate = true; + + kfree(params); + + return ret; +} +EXPORT_SYMBOL_GPL(kvm_rme_dev_assign); + +void kvm_rme_dev_unassign(struct pci_dev *pdev, struct kvm *kvm) +{ + struct rmi_dev_assign_params *params = NULL; + struct pci_dev *root_dev; + int ret; + + if (!is_support_rme() || !pdev || !kvm) + return; + + root_dev = rme_get_root_dev(pdev); + if (!root_dev) + return; + + params = kzalloc(sizeof(*params), GFP_KERNEL); + if (!params) { + pci_err(root_dev, "Failed to alloc mem for params\n"); + return; + } + + ret = rme_get_all_dev_info(root_dev, params); + if (ret) { + kfree(params); + return; + } + + WARN_ON(rmi_dev_unassign(pci_dev_id(root_dev), virt_to_phys(params))); + + kfree(params); +} +EXPORT_SYMBOL_GPL(kvm_rme_dev_unassign); + +static struct pci_dev *find_pdev_by_bdf(u16 dev_bdf) +{ + unsigned int bus, slot, fn; + struct pci_dev *pdev; + + bus = (dev_bdf >> 8) & 0xff; + slot = (dev_bdf >> 3) & 0x1f; + fn = dev_bdf & 0x7; + + /* Only support domain 0 */ + pdev = pci_get_domain_bus_and_slot(0, bus, PCI_DEVFN(slot, fn)); + if (!pdev) { + pr_err("Invalid pci dev bdf %#x\n", dev_bdf); + return NULL; + } + return pdev; +} + +int kvm_arm_vcpu_rme_dev_validate(struct kvm_vcpu *vcpu, + struct kvm_arm_rme_dev_validate *args) +{ + struct pci_dev *pdev; + + if (!vcpu || !args || !_vcpu_is_rec(vcpu)) + return -EINVAL; + + /* Host dev bdf does not refer to a VFIO dev, no need to validate */ + if (!args->vfio_dev) + return 0; + + pdev = find_pdev_by_bdf(args->dev_bdf); + if (!pdev) + return -EINVAL; + + /* Record host pdev bdf in rec, complete dev validate in rec pre enter */ + vcpu->arch.rec->run->exit.dev_bdf = args->dev_bdf; + vcpu->arch.rec->run->exit.valid = true; + + return 0; +} + +void kvm_complete_dev_validate(struct kvm_vcpu *vcpu) +{ + phys_addr_t rd_phys, rec_phys; + u16 dev_bdf, guest_dev_bdf; + struct realm_rec *rec; + struct realm *realm; + struct kvm *kvm; + int ret; + + if (!vcpu || !_vcpu_is_rec(vcpu) || kvm_realm_state(vcpu->kvm) != + REALM_STATE_ACTIVE) + return; + + kvm = vcpu->kvm; + realm = &kvm->arch.realm; + rec = vcpu->arch.rec; + + /* Host dev bdf is invalid, no need to validate */ + if (!rec->run->exit.valid) + return; + + dev_bdf = rec->run->exit.dev_bdf; + guest_dev_bdf = rec->run->exit.guest_dev_bdf; + rec_phys = virt_to_phys(rec->rec_page); + rd_phys = virt_to_phys(realm->rd); + + ret = rmi_dev_validate(rd_phys, rec_phys, dev_bdf, guest_dev_bdf); + /* + * if ret == RMI_ERROR_DEV_INFO, it means the device is not found in RMM, + * which is definitely not realm + */ + WARN_ON(ret && ret != RMI_ERROR_DEV_INFO); +} + +int kvm_rme_dev_attach(struct pci_dev *pdev, struct kvm *kvm) +{ + struct rdev_node *realm_dev; + struct realm *realm; + phys_addr_t rd; + + if (!pdev || !kvm || !is_support_rme()) + return -EINVAL; + + if (!kvm_is_realm(kvm)) + return 0; + + if (kvm_realm_is_created(kvm)) { + realm = &kvm->arch.realm; + rd = virt_to_phys(realm->rd); + return rmi_dev_attach(pci_dev_id(pdev), rd); + } + + /* Add the dev to list first. Once the RD is created, do rmi_dev_attach */ + realm_dev = kzalloc(sizeof(*realm_dev), GFP_KERNEL); + if (!realm_dev) + return -ENOMEM; + + realm_dev->dev_bdf = pci_dev_id(pdev); + list_add_tail(&realm_dev->list, &kvm->arch.realm.rdev_list); + + return 0; +} +EXPORT_SYMBOL_GPL(kvm_rme_dev_attach); + +void kvm_rme_dev_detach(struct pci_dev *pdev, struct kvm *kvm) +{ + if (!pdev || !kvm || !is_support_rme() || !kvm_is_realm(kvm)) + return; + + WARN_ON(rmi_dev_detach(pci_dev_id(pdev))); +} +EXPORT_SYMBOL_GPL(kvm_rme_dev_detach); + +static void realm_free_rdev_list(struct realm *realm) +{ + struct rdev_node *pos, *n; + + list_for_each_entry_safe(pos, n, &realm->rdev_list, list) { + list_del(&pos->list); + kfree(pos); + } +} + +/* After RD created, call this to do attach dev */ +int realm_attach_devs(struct realm *realm) +{ + struct rdev_node *dev; + phys_addr_t rd; + int ret; + + if (!realm || !realm->rd) + return -EINVAL; + + rd = virt_to_phys(realm->rd); + + list_for_each_entry(dev, &realm->rdev_list, list) { + ret = rmi_dev_attach(dev->dev_bdf, rd); + if (ret) + goto err_detach; + } + + realm_free_rdev_list(realm); + return 0; + +err_detach: + list_for_each_entry_continue_reverse(dev, &realm->rdev_list, list) + WARN_ON(rmi_dev_detach(dev->dev_bdf)); + + realm_free_rdev_list(realm); + return ret; +} diff --git a/arch/arm64/kvm/rme-exit.c b/arch/arm64/kvm/rme-exit.c index abe13350d118..68c5acaca82f 100644 --- a/arch/arm64/kvm/rme-exit.c +++ b/arch/arm64/kvm/rme-exit.c @@ -12,6 +12,9 @@ #include #include +#ifdef CONFIG_HISI_CCA_DA +#include +#endif typedef int (*exit_handler_fn)(struct kvm_vcpu *vcpu); static int rec_exit_reason_notimpl(struct kvm_vcpu *vcpu) @@ -131,6 +134,17 @@ static void update_arch_timer_irq_lines(struct kvm_vcpu *vcpu) kvm_realm_timers_update(vcpu); } +static int rec_exit_validate_dev(struct kvm_vcpu *vcpu) +{ + struct realm_rec *rec = vcpu->arch.rec; + + /* Return to qemu to validate dev */ + vcpu->run->exit_reason = KVM_EXIT_ARM_RME_DEV; + vcpu->run->rme_dev.guest_dev_bdf = rec->run->exit.guest_dev_bdf; + + return 0; +} + /* * Return > 0 to return to guest, < 0 on error, 0 (and set exit_reason) on * proper exit to userspace. @@ -188,6 +202,8 @@ int _handle_rec_exit(struct kvm_vcpu *vcpu, int rec_run_ret) return rec_exit_ripas_change(vcpu); case RMI_EXIT_HOST_CALL: return rec_exit_host_call(vcpu); + case RMI_EXIT_VALIDATE_DEV: + return rec_exit_validate_dev(vcpu); } kvm_pr_unimpl("Unsupported exit reason: %u\n", diff --git a/arch/arm64/kvm/rme-pmu.c b/arch/arm64/kvm/rme-pmu.c new file mode 100644 index 000000000000..2fc20cd1c608 --- /dev/null +++ b/arch/arm64/kvm/rme-pmu.c @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: GPL-2.0-only + +#include + +static struct rme_perf_record_unit g_rme_pmu; +struct rme_perf_record_unit *get_rme_pmu(void) +{ + return &g_rme_pmu; +} +EXPORT_SYMBOL_GPL(get_rme_pmu); + +void rme_perf_init(void) +{ + mutex_init(&g_rme_pmu.lock); + memset(&g_rme_pmu, 0, sizeof(struct rme_perf_record_unit)); + g_rme_pmu.report.smc_num = SMC_RECORD_NUM; + g_rme_pmu.report.cntfrq = arch_timer_get_cntfrq(); + g_rme_pmu.start_record = true; +} +EXPORT_SYMBOL_GPL(rme_perf_init); + +void rme_perf_record(int idx, unsigned long duration) +{ + if (!g_rme_pmu.start_record) + return; + + atomic64_add(1, (atomic64_t *)&g_rme_pmu.report.smc_recorder[idx].times); + atomic64_add(duration, (atomic64_t *)&g_rme_pmu.report.smc_recorder[idx].duration); +} +EXPORT_SYMBOL_GPL(rme_perf_record); diff --git a/arch/arm64/kvm/rme.c b/arch/arm64/kvm/rme.c index a7f6a2a2ecb9..751fd430de12 100644 --- a/arch/arm64/kvm/rme.c +++ b/arch/arm64/kvm/rme.c @@ -13,9 +13,18 @@ #include #include #include +#ifdef CONFIG_HISI_CCA_DA +#include +#endif static unsigned long rmm_feat_reg0; +struct undelegate_region { + unsigned long start; + unsigned long size; + struct list_head list; +}; + #define RMM_PAGE_SHIFT 12 #define RMM_PAGE_SIZE BIT(RMM_PAGE_SHIFT) @@ -224,18 +233,81 @@ static int realm_rtt_fold(struct realm *realm, return ret; } +static bool realm_is_addr_io_private(struct realm *realm, unsigned long addr) +{ + phys_addr_t rd = virt_to_phys(realm->rd); + struct rtt_entry rtt; + + if (rmi_rtt_read_entry(rd, addr, RMM_RTT_MAX_LEVEL, &rtt)) + return false; + + if (rtt.ripas == RMI_IO && rtt.state == RMI_ASSIGNED_IO_PRIVATE) + return true; + + return false; +} + +static bool realm_expand_undelegate_regions(struct realm *realm, unsigned long addr) +{ + struct undelegate_region *region; + + list_for_each_entry(region, &realm->undelegate_list, list) { + if (addr == region->start - RMM_PAGE_SIZE) { + region->start = addr; + region->size += RMM_PAGE_SIZE; + return true; + } else if (addr == region->start + region->size) { + region->size += RMM_PAGE_SIZE; + return true; + } + } + return false; +} + +static int realm_add_undelegate_region(struct realm *realm, unsigned long addr) +{ + struct undelegate_region *region; + + region = kzalloc(sizeof(*region), GFP_KERNEL); + if (!region) + return -ENOMEM; + + region->start = addr; + region->size = RMM_PAGE_SIZE; + + list_add_tail(®ion->list, &realm->undelegate_list); + + return 0; +} + +static int realm_add_undelegate_page(struct realm *realm, unsigned long addr) +{ + /* Don't free page until it's undelegated */ + get_page(phys_to_page(addr)); + + if (!realm_expand_undelegate_regions(realm, addr)) + return realm_add_undelegate_region(realm, addr); + return 0; +} + static int realm_destroy_private_granule(struct realm *realm, unsigned long ipa, unsigned long *next_addr, - phys_addr_t *out_rtt) + phys_addr_t *out_rtt, + bool delay_undelegate) { unsigned long rd = virt_to_phys(realm->rd); unsigned long rtt_addr; phys_addr_t rtt; + bool io_private; int ret; + io_private = realm_is_addr_io_private(realm, ipa); retry: - ret = rmi_data_destroy(rd, ipa, &rtt_addr, next_addr); + if (io_private) + ret = rmi_io_destroy(rd, ipa, &rtt_addr, next_addr); + else + ret = rmi_data_destroy(rd, ipa, &rtt_addr, next_addr); if (RMI_RETURN_STATUS(ret) == RMI_ERROR_RTT) { if (*next_addr > ipa) return 0; /* UNASSIGNED */ @@ -258,7 +330,12 @@ static int realm_destroy_private_granule(struct realm *realm, return -ENXIO; } - ret = rmi_granule_undelegate(rtt_addr); + if (io_private) + ret = rmi_granule_io_undelegate(rtt_addr); + else if (delay_undelegate) + ret = realm_add_undelegate_page(realm, rtt_addr); + else + ret = rmi_granule_undelegate(rtt_addr); if (WARN_ON(ret)) return -ENXIO; @@ -269,7 +346,8 @@ static int realm_destroy_private_granule(struct realm *realm, static int realm_unmap_private_page(struct realm *realm, unsigned long ipa, - unsigned long *next_addr) + unsigned long *next_addr, + bool delay_undelegate) { unsigned long end = ALIGN(ipa + 1, PAGE_SIZE); unsigned long addr; @@ -278,7 +356,7 @@ static int realm_unmap_private_page(struct realm *realm, for (addr = ipa; addr < end; addr = *next_addr) { ret = realm_destroy_private_granule(realm, addr, next_addr, - &out_rtt); + &out_rtt, delay_undelegate); if (ret) return ret; } @@ -697,14 +775,16 @@ void kvm_realm_destroy_rtts(struct kvm *kvm, u32 ia_bits) static void realm_unmap_private_range(struct kvm *kvm, unsigned long start, - unsigned long end) + unsigned long end, + bool delay_undelegate) { struct realm *realm = &kvm->arch.realm; unsigned long next_addr, addr; int ret; for (addr = start; addr < end; addr = next_addr) { - ret = realm_unmap_private_page(realm, addr, &next_addr); + ret = realm_unmap_private_page(realm, addr, &next_addr, + delay_undelegate); if (ret) break; @@ -715,7 +795,8 @@ static void realm_unmap_private_range(struct kvm *kvm, } void kvm_realm_unmap_range(struct kvm *kvm, unsigned long start, - unsigned long size, bool unmap_private) + unsigned long size, bool unmap_private, + bool delay_undelegate) { unsigned long end = start + size; struct realm *realm = &kvm->arch.realm; @@ -734,7 +815,7 @@ void kvm_realm_unmap_range(struct kvm *kvm, unsigned long start, realm_unmap_shared_range(kvm, find_map_level(realm, start, end), start, end); if (unmap_private) - realm_unmap_private_range(kvm, start, end); + realm_unmap_private_range(kvm, start, end, delay_undelegate); } static int realm_create_protected_data_granule(struct realm *realm, @@ -942,6 +1023,106 @@ int realm_map_protected(struct realm *realm, return -ENXIO; } +#ifdef CONFIG_HISI_CCA_DA +int realm_map_mmio_protected(struct realm *realm, unsigned long ipa, + kvm_pfn_t pfn, unsigned long map_size, + struct kvm_mmu_memory_cache *memcache) +{ + phys_addr_t phys = __pfn_to_phys(pfn); + phys_addr_t rd = virt_to_phys(realm->rd); + unsigned long base_ipa = ipa; + unsigned long size; + unsigned long data; + unsigned long top; + int map_level; + int level; + int ret = 0; + + if (WARN_ON(!IS_ALIGNED(map_size, RMM_PAGE_SIZE))) + return -EINVAL; + + if (WARN_ON(!IS_ALIGNED(ipa, map_size))) + return -EINVAL; + + if (IS_ALIGNED(map_size, RMM_L2_BLOCK_SIZE)) + map_level = 2; + else + map_level = 3; + + if (map_level < RMM_RTT_MAX_LEVEL) { + /* + * A temporary RTT is needed during the map, precreate it, + * however if there is an error (e.g. missing parent tables) + * this will be handled below. + */ + realm_create_rtt_levels(realm, ipa, map_level, + RMM_RTT_MAX_LEVEL, memcache); + } + + for (size = 0; size < map_size; size += RMM_PAGE_SIZE) { + if (rmi_granule_io_delegate(phys, 0)) { + /* + * It's likely we raced with another VCPU on the same + * fault. Assume the other VCPU has handled the fault + * and return to the guest. + */ + return 0; + } + + ret = rmi_io_create(rd, phys, ipa); + + if (RMI_RETURN_STATUS(ret) == RMI_ERROR_RTT) { + /* Create missing RTTs and retry */ + level = RMI_RETURN_INDEX(ret); + + WARN_ON(level == RMM_RTT_MAX_LEVEL); + + ret = realm_create_rtt_levels(realm, ipa, level, + RMM_RTT_MAX_LEVEL, + memcache); + if (ret) + goto err_undelegate; + + ret = rmi_io_create(rd, phys, ipa); + } + + if (WARN_ON(ret)) + goto err_undelegate; + + phys += RMM_PAGE_SIZE; + ipa += RMM_PAGE_SIZE; + } + + if (map_size == RMM_L2_BLOCK_SIZE) { + ret = fold_rtt(realm, base_ipa, map_level + 1); + if (WARN_ON(ret)) + goto err; + } + + return 0; + +err_undelegate: + if (WARN_ON(rmi_granule_io_undelegate(phys))) { + /* Page can't be returned to NS world so is lost */ + get_page(phys_to_page(phys)); + } +err: + while (size > 0) { + phys -= RMM_PAGE_SIZE; + size -= RMM_PAGE_SIZE; + ipa -= RMM_PAGE_SIZE; + + WARN_ON(rmi_io_destroy(rd, ipa, &data, &top)); + + if (WARN_ON(rmi_granule_io_undelegate(phys))) { + /* Page can't be returned to NS world so is lost */ + get_page(phys_to_page(phys)); + } + } + return -ENXIO; +} +#endif + int realm_map_non_secure(struct realm *realm, unsigned long ipa, kvm_pfn_t pfn, @@ -1189,7 +1370,7 @@ static int realm_set_ipa_state(struct kvm_vcpu *vcpu, return ret; if (ripas == RMI_EMPTY && ipa != start) - realm_unmap_private_range(kvm, start, ipa); + realm_unmap_private_range(kvm, start, ipa, false); return ret; } @@ -1248,6 +1429,42 @@ static int kvm_init_ipa_range_realm(struct kvm *kvm, return realm_init_ipa_state(realm, addr, end); } +static int kvm_map_ram_realm(struct kvm *kvm, + struct arm_rme_map_ram_args *args) +{ + unsigned long ram_base, ram_size, size; + struct realm *realm = &kvm->arch.realm; + struct kvm_memory_slot *memslot; + kvm_pfn_t pfn; + gfn_t gfn; + int ret; + + ram_base = args->ram_base; + ram_size = args->ram_size; + + if (ram_base >= ram_base + ram_size) + return -EINVAL; + + for (size = 0; size < ram_size; size += PAGE_SIZE) { + gfn = gpa_to_gfn(ram_base + size); + memslot = gfn_to_memslot(kvm, gfn); + if (!memslot) { + return -EFAULT; + } + pfn = gfn_to_pfn_memslot(memslot, gfn); + if (is_error_pfn(pfn)) { + return -EFAULT; + } + ret = realm_map_protected(realm, ram_base + size, pfn, PAGE_SIZE, NULL); + if (ret) { + kvm_release_pfn_clean(pfn); + break; + } + kvm_release_pfn_dirty(pfn); + } + return ret; +} + static int kvm_activate_realm(struct kvm *kvm) { struct realm *realm = &kvm->arch.realm; @@ -1325,6 +1542,15 @@ static int kvm_create_realm(struct kvm *kvm) free_page((unsigned long)realm->params); realm->params = NULL; +#ifdef CONFIG_HISI_CCA_DA + ret = realm_attach_devs(realm); + if (ret) { + kvm_err("Fail to attach devs\n"); + kvm_destroy_realm(kvm); + return ret; + } +#endif + return 0; } @@ -1417,6 +1643,18 @@ int _kvm_realm_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) case KVM_CAP_ARM_RME_ACTIVATE_REALM: r = kvm_activate_realm(kvm); break; + case KVM_CAP_ARM_RME_MAP_RAM_REALM: { + struct arm_rme_map_ram_args args; + void __user *argp = u64_to_user_ptr(cap->args[1]); + + if (copy_from_user(&args, argp, sizeof(args))) { + r = -EFAULT; + break; + } + + r = kvm_map_ram_realm(kvm, &args); + break; + } case KVM_CAP_ARM_RME_MAP_RAM_CCAL: { struct arm_rme_populate_realm args; void __user *argp = u64_to_user_ptr(cap->args[1]); @@ -1437,6 +1675,23 @@ int _kvm_realm_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) return r; } +static void realm_undelegate_all_regions(struct realm *realm) +{ + struct undelegate_region *region, *tmp; + unsigned long addr; + + list_for_each_entry_safe(region, tmp, &realm->undelegate_list, list) { + for (addr = region->start; addr < region->start + region->size; + addr += RMM_PAGE_SIZE) { + if (!WARN_ON(rmi_granule_undelegate(addr))) + put_page(phys_to_page(addr)); + } + /* Now that region is undelegated, we don't need this anymore */ + list_del(®ion->list); + kfree(region); + } +} + void _kvm_destroy_realm(struct kvm *kvm) { struct realm *realm = &kvm->arch.realm; @@ -1475,6 +1730,9 @@ void _kvm_destroy_realm(struct kvm *kvm) /* Now that the Realm is destroyed, free the entry level RTTs */ kvm_free_stage2_pgd(&kvm->arch.mmu); + + /* Undelegate all realm memory */ + realm_undelegate_all_regions(realm); } static void kvm_complete_ripas_change(struct kvm_vcpu *vcpu) @@ -1529,6 +1787,11 @@ int _kvm_rec_pre_enter(struct kvm_vcpu *vcpu) case RMI_EXIT_RIPAS_CHANGE: kvm_complete_ripas_change(vcpu); break; +#ifdef CONFIG_HISI_CCA_DA + case RMI_EXIT_VALIDATE_DEV: + kvm_complete_dev_validate(vcpu); + break; +#endif } return 1; @@ -1538,6 +1801,8 @@ int _kvm_rec_enter(struct kvm_vcpu *vcpu) { struct realm_rec *rec = vcpu->arch.rec; + rec->run->enter.clidr_el1 = vcpu_read_sys_reg(vcpu, CLIDR_EL1); + return rmi_rec_enter(virt_to_phys(rec->rec_page), virt_to_phys(rec->run)); } @@ -1731,6 +1996,12 @@ int _kvm_init_realm_vm(struct kvm *kvm) if (!kvm->arch.realm.params) return -ENOMEM; + + INIT_LIST_HEAD(&kvm->arch.realm.undelegate_list); + +#ifdef CONFIG_HISI_CCA_DA + INIT_LIST_HEAD(&kvm->arch.realm.rdev_list); +#endif return 0; } @@ -1739,7 +2010,9 @@ void _kvm_init_rme(void) if (PAGE_SIZE != SZ_4K) /* Only 4k page size on the host is supported */ return; - +#ifdef CONFIG_CCA_RME_PMU + RMI_PERF_INIT(); +#endif if (rmi_check_version()) /* Continue without realm support */ return; diff --git a/drivers/base/core.c b/drivers/base/core.c index c7bd7ce2e8aa..5be6c22158f4 100644 --- a/drivers/base/core.c +++ b/drivers/base/core.c @@ -34,6 +34,7 @@ #include #include /* for dma_default_coherent */ #include +#include #if defined(CONFIG_PSWIOTLB) && !defined(__GENKSYMS__) #include @@ -3149,6 +3150,7 @@ void device_initialize(struct device *dev) #endif swiotlb_dev_init(dev); enable_swiotlb_for_cvm_dev(dev, false); + enable_swiotlb_for_realm_dev(dev, false); #ifdef CONFIG_PSWIOTLB if ((pswiotlb_force_disable != true) && is_phytium_ps_socs()) diff --git a/drivers/base/platform-msi.c b/drivers/base/platform-msi.c index 3348d4db5f1b..d94e57037990 100644 --- a/drivers/base/platform-msi.c +++ b/drivers/base/platform-msi.c @@ -221,7 +221,7 @@ int platform_msi_domain_alloc_irqs(struct device *dev, unsigned int nvec, } EXPORT_SYMBOL_GPL(platform_msi_domain_alloc_irqs); -#ifdef CONFIG_HISI_VIRTCCA_CODA +#if defined(CONFIG_HISI_VIRTCCA_CODA) || defined(CONFIG_HISI_CCA_DA) /** * platform_msi_domain_alloc_range_irqs - Allocate specific scope MSI interrupts for @dev * @dev: The device for which to allocate interrupts diff --git a/drivers/iommu/Kconfig b/drivers/iommu/Kconfig index 56eafa478c34..7d426d6124c7 100644 --- a/drivers/iommu/Kconfig +++ b/drivers/iommu/Kconfig @@ -412,6 +412,15 @@ config ARM_SMMU_V3_SVA Say Y here if your system supports SVA extensions such as PCIe PASID and PRI. +config HISI_CCA_DA + bool "Support for hisilicon CCA device assignment" + depends on ARM_SMMU_V3 + default y + help + Support for hisilicon CCA device assignment + + If unsure, say Y. + config ARM_SMMU_V3_PM bool "Add arm_smmu_v3 suspend and resume support" depends on ARM_SMMU_V3 && PM_SLEEP diff --git a/drivers/iommu/Makefile b/drivers/iommu/Makefile index e3ecb2040808..a32cc54d426f 100644 --- a/drivers/iommu/Makefile +++ b/drivers/iommu/Makefile @@ -30,5 +30,6 @@ obj-$(CONFIG_IOMMU_SVA) += iommu-sva.o obj-$(CONFIG_IOMMU_IOPF) += io-pgfault.o obj-$(CONFIG_SPRD_IOMMU) += sprd-iommu.o obj-$(CONFIG_APPLE_DART) += apple-dart.o +obj-$(CONFIG_HISI_CCA_DA) += io-pgtable-realm.o obj-$(CONFIG_LOONGARCH_IOMMU) += loongarch_iommu.o diff --git a/drivers/iommu/arm/arm-smmu-v3/Makefile b/drivers/iommu/arm/arm-smmu-v3/Makefile index 3430d2c82b90..49683374d7d9 100644 --- a/drivers/iommu/arm/arm-smmu-v3/Makefile +++ b/drivers/iommu/arm/arm-smmu-v3/Makefile @@ -1,6 +1,7 @@ # SPDX-License-Identifier: GPL-2.0 obj-$(CONFIG_ARM_SMMU_V3) += arm_smmu_v3.o arm_smmu_v3-y := arm-smmu-v3.o +arm_smmu_v3-$(CONFIG_HISI_CCA_DA) += arm-r-smmu-v3.o arm_smmu_v3-$(CONFIG_HISI_VIRTCCA_CODA) += arm-s-smmu-v3.o arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_IOMMUFD) += arm-smmu-v3-iommufd.o arm_smmu_v3-$(CONFIG_ARM_SMMU_V3_SVA) += arm-smmu-v3-sva.o diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.c new file mode 100644 index 000000000000..06a5badc1fce --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.c @@ -0,0 +1,1215 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "arm-smmu-v3.h" +#include "arm-r-smmu-v3.h" + +/* + * Add realm IRQs index based on arm_smmu_msi_index + * Struct arm_r_smmu_msi_index need keep same as struct arm_smmu_msi_index + */ +enum arm_r_smmu_msi_index { + EVTQ_MSI_INDEX, + GERROR_MSI_INDEX, + PRIQ_MSI_INDEX, + R_EVTQ_MSI_INDEX, + R_GERROR_MSI_INDEX, + ARM_R_SMMU_MAX_MSIS, +}; + +#define ARM_R_SMMU_MAX_CFGS 0x3 + +enum smmu_config { + SMMU_STRTAB_INIT, + SMMU_STRTAB_DEINIT, + SMMU_STRTAB_L2_INIT, + SMMU_STRTAB_L2_DEINIT, + SMMU_QUEUE_INIT, + SMMU_QUEUE_DEINIT, + SMMU_CDTAB_INIT, + SMMU_CDTAB_DEINIT, + SMMU_CDTAB_L2_INIT, + SMMU_CDTAB_L2_DEINIT, +}; + +struct cdtab_l1_params { + unsigned long ioaddr; + unsigned long cdtab_pa; + unsigned long cdtab_size; + unsigned long fmt; +}; + +struct cdtab_l2_params { + unsigned long ioaddr; + unsigned long cdtab_pa; + unsigned long ssid; + unsigned long l2_pa; + unsigned long l2_size; +}; + +struct strtab_params { + unsigned long ioaddr; + unsigned long strtab_base; + unsigned long strtab_base_cfg; +}; + +struct strtab_l2_params { + unsigned long ioaddr; + unsigned long strtab_base; + unsigned long sid; + unsigned long l2_pa; + unsigned long span; +}; + +struct queue_params { + unsigned long ioaddr; + unsigned long queue_type; + unsigned long q_base; + unsigned long q_log2; +}; + +#define realm_smmu_read_poll_timeout(addr, val, cond, delay_us, timeout_us) \ + realm_read_poll_timeout(rmi_smmu_reg_read32, val, cond, delay_us, \ + timeout_us, false, smmu->realm.ioaddr, addr) + +/* Add realm IRQs cfg based on arm_smmu_msi_cfg */ +static phys_addr_t arm_r_smmu_msi_cfg[ARM_R_SMMU_MAX_MSIS][ARM_R_SMMU_MAX_CFGS] = { + [R_EVTQ_MSI_INDEX] = { + SMMU_R_EVENTQ_IRQ_CFG0, + SMMU_R_EVENTQ_IRQ_CFG1, + SMMU_R_EVENTQ_IRQ_CFG2, + }, + [R_GERROR_MSI_INDEX] = { + SMMU_R_GERROR_IRQ_CFG0, + SMMU_R_GERROR_IRQ_CFG1, + SMMU_R_GERROR_IRQ_CFG2, + }, +}; + +bool arm_smmu_support_rme(struct arm_smmu_device *smmu) +{ + return (smmu->realm.rme_features & ARM_SMMU_FEAT_RME) && smmu->realm.enabled; +} + +static int realm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val, + unsigned int reg_off, unsigned int ack_off) +{ + int ret; + u32 reg; + + ret = rmi_smmu_reg_write32(smmu->realm.ioaddr, reg_off, val); + if (ret) + return ret; + + return realm_smmu_read_poll_timeout(ack_off, reg, reg == val, 1, + ARM_SMMU_POLL_TIMEOUT_US); +} + +#define GRANULE_ALIGN_UP(x) (((x) + (GRANULE_DELEGATE_SIZE) - 1) & ((~(GRANULE_DELEGATE_SIZE)) - 1)) + +static int realm_smmu_config(unsigned long config, void *params, unsigned long size) +{ + void *ns_params; + int ret; + + ns_params = (void *)get_zeroed_page(GFP_KERNEL); + if (!ns_params) + return -ENOMEM; + + if (size > PAGE_SIZE) { + free_page((unsigned long)ns_params); + return -EINVAL; + } + + memcpy(ns_params, params, size); + ret = rmi_smmu_config(config, virt_to_phys(ns_params)); + + free_page((unsigned long)ns_params); + + return ret; +} + +static int realm_config_cdtab(enum smmu_config config, unsigned long ioaddr, + unsigned long cdtab_pa, unsigned long cdtab_size, + unsigned long fmt) +{ + struct cdtab_l1_params params = { + .ioaddr = ioaddr, + .cdtab_pa = cdtab_pa, + .cdtab_size = cdtab_size, + .fmt = fmt, + }; + + return realm_smmu_config(config, (void *)¶ms, sizeof(params)); +} + +static int realm_config_cdtab_l2(enum smmu_config config, unsigned long ioaddr, + unsigned long cdtab_pa, unsigned long ssid, + unsigned long l2_pa, unsigned long l2_size) +{ + struct cdtab_l2_params params = { + .ioaddr = ioaddr, + .cdtab_pa = cdtab_pa, + .ssid = ssid, + .l2_pa = l2_pa, + .l2_size = l2_size, + }; + + return realm_smmu_config(config, (void *)¶ms, sizeof(params)); +} + +static int realm_config_strtab(enum smmu_config config, unsigned long ioaddr, + unsigned long strtab_base, + unsigned long strtab_base_cfg) +{ + struct strtab_params params = { + .ioaddr = ioaddr, + .strtab_base = strtab_base, + .strtab_base_cfg = strtab_base_cfg, + }; + + return realm_smmu_config(config, (void *)¶ms, sizeof(params)); +} + +static int realm_config_strtab_l2(enum smmu_config config, unsigned long ioaddr, + unsigned long strtab_base, unsigned long sid, + unsigned long l2_pa, unsigned long span) +{ + struct strtab_l2_params params = { + .ioaddr = ioaddr, + .strtab_base = strtab_base, + .sid = sid, + .l2_pa = l2_pa, + .span = span, + }; + + return realm_smmu_config(config, (void *)¶ms, sizeof(params)); +} + +static int realm_config_queue(enum smmu_config config, unsigned long ioaddr, + unsigned long queue_type, unsigned long q_base, + unsigned long q_log2) +{ + struct queue_params params = { + .ioaddr = ioaddr, + .queue_type = queue_type, + .q_base = q_base, + .q_log2 = q_log2, + }; + + return realm_smmu_config(config, (void *)¶ms, sizeof(params)); +} + +static int realm_smmu_alloc_cd_leaf_table(struct arm_smmu_master *master, + u32 ssid) +{ + int ret; + size_t align_size; + struct arm_smmu_cd *l2ptr; + struct arm_smmu_l1_ctx_desc *l1_desc; + struct arm_smmu_device *smmu = master->smmu; + size_t size = CTXDESC_L2_ENTRIES * sizeof(struct arm_smmu_cd); + struct realm_smmu_ctx_desc_cfg *cd_table = &master->realm_cd_table; + + if (!cd_table->l1_desc) + return 0; + + l1_desc = &cd_table->l1_desc[ssid >> CTXDESC_SPLIT]; + if (l1_desc->l2ptr) + return 0; + + align_size = GRANULE_ALIGN_UP(size); + l2ptr = dma_alloc_coherent(smmu->dev, align_size, &l1_desc->l2ptr_dma, + GFP_KERNEL); + if (!l2ptr) { + dev_warn(smmu->dev, + "failed to allocate realm context descriptor table\n"); + return -ENOMEM; + } + + ret = granule_delegate_range(l1_desc->l2ptr_dma, size); + if (ret) + goto out_free; + + ret = realm_config_cdtab_l2(SMMU_CDTAB_L2_INIT, smmu->realm.ioaddr, + cd_table->cdtab_dma, ssid, + l1_desc->l2ptr_dma, size); + if (ret) + goto out_undelegate; + + l1_desc->l2ptr = l2ptr; + return 0; + +out_undelegate: + if (WARN_ON(granule_undelegate_range(l1_desc->l2ptr_dma, size))) + return ret; +out_free: + dma_free_coherent(smmu->dev, align_size, l2ptr, l1_desc->l2ptr_dma); + return ret; +} + +static int realm_smmu_alloc_cd_tables(struct arm_smmu_master *master) +{ + int ret; + __le64 *l1_desc; + size_t l1size, align_size; + struct arm_smmu_device *smmu = master->smmu; + struct realm_smmu_ctx_desc_cfg *cd_table = &master->realm_cd_table; + + if (cd_table->cdtab.l1_desc) + return 0; + + cd_table->s1fmt = master->cd_table.s1fmt; + + if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR) { + cd_table->num_l1_ents = master->cd_table.linear.num_ents; + l1size = cd_table->num_l1_ents * sizeof(struct arm_smmu_cd); + } else { + cd_table->num_l1_ents = master->cd_table.l2.num_l1_ents; + l1size = cd_table->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); + cd_table->l1_desc = kcalloc(cd_table->num_l1_ents, + sizeof(*cd_table->l1_desc), + GFP_KERNEL); + if (!cd_table->l1_desc) + return -ENOMEM; + } + + align_size = GRANULE_ALIGN_UP(l1size); + l1_desc = dma_alloc_coherent(smmu->dev, align_size, &cd_table->cdtab_dma, + GFP_KERNEL); + if (!l1_desc) { + ret = -ENOMEM; + goto out_free_l1_desc; + } + + ret = granule_delegate_range(cd_table->cdtab_dma, l1size); + if (ret) + goto out_free_dma; + + ret = realm_config_cdtab(SMMU_CDTAB_INIT, smmu->realm.ioaddr, + cd_table->cdtab_dma, l1size, cd_table->s1fmt); + if (ret) + goto out_undelegate; + + cd_table->cdtab.l1_desc = l1_desc; + return 0; + +out_undelegate: + if (WARN_ON(granule_undelegate_range(cd_table->cdtab_dma, l1size))) + return ret; +out_free_dma: + dma_free_coherent(smmu->dev, align_size, cd_table->cdtab.l1_desc, + cd_table->cdtab_dma); +out_free_l1_desc: + if (cd_table->s1fmt != STRTAB_STE_0_S1FMT_LINEAR) + kfree(cd_table->l1_desc); + return ret; +} + +static bool pcipc_ns(struct arm_smmu_master *master) +{ + return !master->realm && master->pcipc_ns_dev; +} + +static bool smmu_need_rme(struct arm_smmu_master *master) +{ + struct arm_smmu_device *smmu = master->smmu; + + if (!arm_smmu_support_rme(smmu)) + return false; + + return master->realm || master->pcipc_ns_dev; +} + +static int realm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid) +{ + int ret; + size_t size; + struct arm_smmu_ste *l2ptr; + struct realm_smmu_strtab_cfg *cfg = &smmu->realm.strtab_cfg; + struct arm_smmu_strtab_l1_desc *desc = &cfg->l1_desc[sid >> STRTAB_SPLIT]; + + if (desc->l2ptr) + return 0; + + size = (1 << STRTAB_SPLIT) * sizeof(struct arm_smmu_ste); + desc->span = STRTAB_SPLIT + 1; + + l2ptr = dma_alloc_coherent(smmu->dev, size, &desc->l2ptr_dma, GFP_KERNEL); + if (!l2ptr) { + dev_err(smmu->dev, + "failed to allocate l2 stream table for SID %u\n", + sid); + return -ENOMEM; + } + + ret = granule_delegate_range(desc->l2ptr_dma, size); + if (ret) { + dev_err(smmu->dev, + "failed to delegate realm l2 stream table for SID %u\n", + sid); + goto out_free; + } + + ret = realm_config_strtab_l2(SMMU_STRTAB_L2_INIT, smmu->realm.ioaddr, + cfg->strtab_dma, sid, desc->l2ptr_dma, + desc->span); + if (ret) { + dev_err(smmu->dev, + "failed to init realm l2 stream table for SID %u\n", + sid); + goto out_undelegate; + } + desc->l2ptr = l2ptr; + return 0; + +out_undelegate: + if (WARN_ON(granule_undelegate_range(desc->l2ptr_dma, size))) + return ret; +out_free: + dma_free_coherent(smmu->dev, size, l2ptr, desc->l2ptr_dma); + return ret; +} + +void realm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, + const struct arm_smmu_ste *target) +{ + u64 config; + bool lvl_strtab; + struct arm_smmu_ste *rste; + struct arm_smmu_device *smmu = master->smmu; + dma_addr_t cdtab_dma = master->realm_cd_table.cdtab_dma; + + if (!smmu_need_rme(master)) + return; + + if (realm_smmu_init_l2_strtab(smmu, sid)) + return; + + rste = (struct arm_smmu_ste *)get_zeroed_page(GFP_KERNEL); + if (!rste) { + dev_err(smmu->dev, "failed to allocate realm ste page\n"); + return; + } + + lvl_strtab = !!(smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB); + memcpy(rste, target, sizeof(struct arm_smmu_ste)); + + /* We should mask S1STALLD in realm */ + rste->data[1] &= ~STRTAB_STE_1_S1STALLD; + + if (pcipc_ns(master) && (rste->data[3] & STRTAB_STE_3_S2TTB_MASK)) + rste->data[3] = master->ns_ttbr & STRTAB_STE_3_S2TTB_MASK; + + /* Replace CD table addr in Stage-1 Translation */ + config = FIELD_GET(STRTAB_STE_0_CFG, rste->data[0]); + if (config == STRTAB_STE_0_CFG_S1_TRANS) { + rste->data[0] &= ~STRTAB_STE_0_S1CTXPTR_MASK; + rste->data[0] |= cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK; + } + + if (rmi_smmu_ste_write(smmu->realm.strtab_cfg.strtab_dma, sid, + virt_to_phys(rste), lvl_strtab)) + dev_err(smmu->dev, "failed to write realm ste\n"); + + free_page((unsigned long)rste); +} + +void realm_smmu_free_cd_tables(struct arm_smmu_master *master) +{ + int i; + size_t size, l1size, align_size; + struct arm_smmu_l1_ctx_desc *l1_desc; + struct arm_smmu_device *smmu = master->smmu; + struct realm_smmu_ctx_desc_cfg *cd_table = &master->realm_cd_table; + + if (!smmu_need_rme(master)) + return; + + if (!cd_table->cdtab.linear) + return; + + if (cd_table->l1_desc) { + size = CTXDESC_L2_ENTRIES * sizeof(struct arm_smmu_cd); + + for (i = 0; i < cd_table->num_l1_ents; i++) { + l1_desc = &cd_table->l1_desc[i]; + if (!l1_desc->l2ptr) + continue; + align_size = GRANULE_ALIGN_UP(size); + (void)realm_config_cdtab_l2(SMMU_CDTAB_L2_DEINIT, + smmu->realm.ioaddr, + cd_table->cdtab_dma, i, + l1_desc->l2ptr_dma, + align_size); + if (WARN_ON(granule_undelegate_range(l1_desc->l2ptr_dma, size))) + continue; + dma_free_coherent(smmu->dev, size, l1_desc->l2ptr, + l1_desc->l2ptr_dma); + } + kfree(cd_table->l1_desc); + cd_table->l1_desc = NULL; + l1size = cd_table->num_l1_ents * (CTXDESC_L1_DESC_DWORDS << 3); + } else { + l1size = cd_table->num_l1_ents * sizeof(struct arm_smmu_cd); + } + + (void)realm_config_cdtab(SMMU_CDTAB_DEINIT, smmu->realm.ioaddr, + cd_table->cdtab_dma, l1size, cd_table->s1fmt); + + + if (WARN_ON(granule_undelegate_range(cd_table->cdtab_dma, l1size))) + return; + align_size = GRANULE_ALIGN_UP(l1size); + dma_free_coherent(smmu->dev, align_size, cd_table->cdtab.l1_desc, + cd_table->cdtab_dma); + cd_table->cdtab.l1_desc = NULL; +} + +static bool is_dev_attach(struct arm_smmu_domain *smmu_domain) +{ + return smmu_domain->realm || smmu_domain->pcipc_ns_dev; +} + +static bool is_dev_detach(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master *master) +{ + if (!smmu_domain->realm && master->realm) + return true; + else if (!smmu_domain->pcipc_ns_dev && master->pcipc_ns_dev) + return true; + else + return false; +} + +void realm_smmu_attach_dev(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master *master) +{ + int i, j; + bool attach; + struct arm_smmu_device *smmu = master->smmu; + struct realm_smmu_device *realm = &smmu->realm; + + if (!arm_smmu_support_rme(smmu)) + return; + + if (is_dev_attach(smmu_domain)) { + master->realm = smmu_domain->realm; + master->pcipc_ns_dev = smmu_domain->pcipc_ns_dev; + smmu->realm.forward_cmd = true; + attach = true; + } else if (is_dev_detach(smmu_domain, master)) { + master->realm = false; + master->pcipc_ns_dev = false; + attach = false; + } else { + return; + } + + write_lock(&realm->fwd_lock); + for (i = 0; i < master->num_streams; i++) { + u32 sid = master->streams[i].id; + /* Bridged PCI devices may end up with duplicated IDs */ + for (j = 0; j < i; j++) + if (master->streams[j].id == sid) + break; + if (j < i) + continue; + + if (attach) + bitmap_set(realm->sid_bitmap, sid, 1); + else + bitmap_clear(realm->sid_bitmap, sid, 1); + } + + if (attach) { + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + bitmap_set(realm->asid_bitmap, smmu_domain->cd.asid, 1); + else + bitmap_set(realm->vmid_bitmap, smmu_domain->s2_cfg.vmid, 1); + } + + write_unlock(&realm->fwd_lock); +} + +void realm_smmu_domain_clear(struct arm_smmu_domain *smmu_domain) +{ + struct arm_smmu_device *smmu = smmu_domain->smmu; + + if (!arm_smmu_support_rme(smmu)) + return; + + write_lock(&smmu->realm.fwd_lock); + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + bitmap_clear(smmu->realm.asid_bitmap, smmu_domain->cd.asid, 1); + else + bitmap_clear(smmu->realm.vmid_bitmap, smmu_domain->s2_cfg.vmid, 1); + write_unlock(&smmu->realm.fwd_lock); +} + +static bool is_realm_sid(struct arm_smmu_device *smmu, u32 sid) +{ + struct realm_smmu_device *realm = &smmu->realm; + bool ret; + + read_lock(&realm->fwd_lock); + ret = bitmap_read(realm->sid_bitmap, sid, 1); + read_unlock(&realm->fwd_lock); + + return ret; +} + +static bool is_realm_vmid(struct arm_smmu_device *smmu, u32 vmid) +{ + struct realm_smmu_device *realm = &smmu->realm; + bool ret; + + read_lock(&realm->fwd_lock); + ret = bitmap_read(realm->vmid_bitmap, vmid, 1); + read_unlock(&realm->fwd_lock); + + return ret; +} + +static bool is_realm_asid(struct arm_smmu_device *smmu, u32 asid) +{ + struct realm_smmu_device *realm = &smmu->realm; + bool ret; + + read_lock(&realm->fwd_lock); + ret = bitmap_read(realm->asid_bitmap, asid, 1); + read_unlock(&realm->fwd_lock); + + return ret; +} + +static bool realm_smmu_need_forward(struct arm_smmu_device *smmu, u64 cmd0, u64 cm1) +{ + u64 opcode = FIELD_GET(CMDQ_0_OP, cmd0); + + switch (opcode) { + case CMDQ_OP_TLBI_EL2_ALL: + case CMDQ_OP_TLBI_NSNH_ALL: + return true; + case CMDQ_OP_PREFETCH_CFG: + case CMDQ_OP_CFGI_CD: + case CMDQ_OP_CFGI_STE: + case CMDQ_OP_CFGI_CD_ALL: + return is_realm_sid(smmu, FIELD_GET(CMDQ_CFGI_0_SID, cmd0)); + case CMDQ_OP_CFGI_ALL: + return true; + case CMDQ_OP_TLBI_NH_VA: + case CMDQ_OP_TLBI_S2_IPA: + case CMDQ_OP_TLBI_NH_ASID: + case CMDQ_OP_TLBI_S12_VMALL: + return is_realm_vmid(smmu, FIELD_GET(CMDQ_TLBI_0_VMID, cmd0)); + case CMDQ_OP_TLBI_EL2_VA: + case CMDQ_OP_TLBI_EL2_ASID: + return is_realm_asid(smmu, FIELD_GET(CMDQ_TLBI_0_ASID, cmd0)); + case CMDQ_OP_ATC_INV: + return is_realm_sid(smmu, FIELD_GET(CMDQ_ATC_0_SID, cmd0)); + case CMDQ_OP_PRI_RESP: + return is_realm_sid(smmu, FIELD_GET(CMDQ_PRI_0_SID, cmd0)); + case CMDQ_OP_RESUME: + return is_realm_sid(smmu, FIELD_GET(CMDQ_RESUME_0_SID, cmd0)); + case CMDQ_OP_CMD_SYNC: + return false; + default: + break; + } + + return false; +} + +int realm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, u64 *cmds, + int n, bool sync) +{ + int i, ret; + u64 *rcmds; + u64 cmd0, cmd1; + u32 forward_cnt = 0; + + if (!arm_smmu_support_rme(smmu)) + return 0; + + if (n * CMDQ_ENT_DWORDS * sizeof(*cmds) > PAGE_SIZE) { + dev_err(smmu->dev, "cmdq batch too large\n"); + return -EINVAL; + } + + rcmds = (u64 *)get_zeroed_page(GFP_KERNEL); + if (!rcmds) + return -ENOMEM; + + for (i = 0; i < n; i++) { + cmd0 = cmds[i * CMDQ_ENT_DWORDS]; + cmd1 = cmds[i * CMDQ_ENT_DWORDS + 1]; + if (realm_smmu_need_forward(smmu, cmd0, cmd1)) { + rcmds[forward_cnt * CMDQ_ENT_DWORDS] = cmd0; + rcmds[forward_cnt * CMDQ_ENT_DWORDS + 1] = cmd1; + forward_cnt++; + } + } + + if (!forward_cnt) { + free_page((unsigned long)rcmds); + return 0; + } + + ret = rmi_smmu_send_cmdlist(smmu->realm.ioaddr, virt_to_phys(rcmds), + forward_cnt, sync); + if (ret) + dev_err(smmu->dev, "issue realm cmd failed\n"); + + free_page((unsigned long)rcmds); + + return ret; +} + +static int realm_smmu_init_queue(struct arm_smmu_device *smmu, + struct realm_smmu_queue *q, int ent_dwords, + int queue_type, const char *name) +{ + __le64 *base; + size_t qsz; + int ret; + + do { + qsz = ((1 << q->max_n_shift) * ent_dwords) << 3; + base = dma_alloc_coherent(smmu->dev, qsz, &q->base_dma, + GFP_KERNEL); + if (base || qsz < PAGE_SIZE) + break; + + q->max_n_shift--; + } while (1); + + if (!base) { + dev_err(smmu->dev, + "failed to allocate queue (0x%zx bytes) for %s\n", + qsz, name); + return -ENOMEM; + } + + if (!WARN_ON(q->base_dma & (qsz - 1))) { + dev_info(smmu->dev, "allocated %u entries for %s\n", + 1 << q->max_n_shift, name); + } + + ret = granule_delegate_range(q->base_dma, qsz); + if (ret) + goto out_free; + + ret = realm_config_queue(SMMU_QUEUE_INIT, smmu->realm.ioaddr, + queue_type, q->base_dma, q->max_n_shift); + if (ret) + goto out_undelegate; + + q->base = base; + return 0; + +out_undelegate: + if (WARN_ON(granule_undelegate_range(q->base_dma, qsz))) + return ret; +out_free: + dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma); + return ret; +} + +static int realm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu) +{ + int ret; + void *strtab; + struct realm_smmu_strtab_cfg *cfg = &smmu->realm.strtab_cfg; + size_t l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3); + + strtab = dma_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, + GFP_KERNEL); + if (!strtab) { + dev_err(smmu->dev, + "failed to alloc rme l1 stream table (%zu bytes)\n", + l1size); + return -ENOMEM; + } + + ret = granule_delegate_range(cfg->strtab_dma, l1size); + if (ret) + goto out_free_strtab; + + cfg->l1_desc = kcalloc(cfg->num_l1_ents, sizeof(*cfg->l1_desc), + GFP_KERNEL); + if (!cfg->l1_desc) { + ret = -ENOMEM; + goto out_undelegate; + } + + ret = realm_config_strtab(SMMU_STRTAB_INIT, smmu->realm.ioaddr, + cfg->strtab_dma, cfg->strtab_base_cfg); + if (ret) { + dev_err(smmu->dev, "failed to init realm linear strtab\n"); + goto out_free_l1_desc; + } + + cfg->strtab.l1_desc = strtab; + + return 0; + +out_free_l1_desc: + kfree(cfg->l1_desc); +out_undelegate: + if (WARN_ON(granule_undelegate_range(cfg->strtab_dma, l1size))) + return ret; +out_free_strtab: + dma_free_coherent(smmu->dev, l1size, strtab, cfg->strtab_dma); + return ret; +} + +static int realm_smmu_init_strtab_linear(struct arm_smmu_device *smmu) +{ + int ret; + void *strtab; + struct realm_smmu_strtab_cfg *cfg = &smmu->realm.strtab_cfg; + size_t l1size = (1 << smmu->sid_bits) * sizeof(cfg->strtab.linear[0]); + + strtab = dma_alloc_coherent(smmu->dev, l1size, &cfg->strtab_dma, + GFP_KERNEL); + if (!strtab) { + dev_err(smmu->dev, + "failed to allocate linear stream table (%zu bytes)\n", + l1size); + return -ENOMEM; + } + + ret = granule_delegate_range(cfg->strtab_dma, l1size); + if (ret) + goto out_free; + + ret = realm_config_strtab(SMMU_STRTAB_INIT, smmu->realm.ioaddr, + cfg->strtab_dma, cfg->strtab_base_cfg); + if (ret) { + dev_err(smmu->dev, "failed to init realm linear strtab\n"); + goto out_undelegate; + } + + cfg->strtab.linear = strtab; + return 0; + +out_undelegate: + if (WARN_ON(granule_undelegate_range(cfg->strtab_dma, l1size))) + return ret; +out_free: + dma_free_coherent(smmu->dev, l1size, strtab, cfg->strtab_dma); + return ret; +} + +static int realm_smmu_init_strtab(struct arm_smmu_device *smmu) +{ + int ret; + u32 reg; + struct realm_smmu_strtab_cfg *cfg = &smmu->realm.strtab_cfg; + + /* Reuse num_l1_ents and strtab_base_cfg */ + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + cfg->num_l1_ents = smmu->strtab_cfg.l2.num_l1_ents; + reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, + STRTAB_BASE_CFG_FMT_2LVL) | + FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, + ilog2(cfg->num_l1_ents) + STRTAB_SPLIT) | + FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT); + cfg->strtab_base_cfg = reg; + ret = realm_smmu_init_strtab_2lvl(smmu); + } else { + cfg->num_l1_ents = smmu->strtab_cfg.linear.num_ents; + reg = FIELD_PREP(STRTAB_BASE_CFG_FMT, + STRTAB_BASE_CFG_FMT_LINEAR) | + FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits); + cfg->strtab_base_cfg = reg; + ret = realm_smmu_init_strtab_linear(smmu); + } + + return ret; +} + +static irqreturn_t arm_r_smmu_revtq_thread(int irq, void *dev) +{ + int i, ret; + struct arm_smmu_device *smmu = dev; + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + u64 evt[EVTQ_ENT_DWORDS]; + u8 id; + + do { + ret = rmi_smmu_read_event(smmu->realm.ioaddr, evt); + if (ret) + break; + + id = FIELD_GET(EVTQ_0_ID, evt[0]); + if (!__ratelimit(&rs)) + continue; + + dev_info(smmu->dev, "event 0x%02x received:\n", id); + for (i = 0; i < ARRAY_SIZE(evt); ++i) + dev_info(smmu->dev, "\t0x%016llx\n", + (unsigned long long)evt[i]); + cond_resched(); + } while (true); + + return IRQ_HANDLED; +} + +static irqreturn_t arm_smmu_realm_gerror_handler(int irq, void *dev) +{ + u32 gerror, gerrorn, active; + struct arm_smmu_device *smmu = dev; + int ret; + + ret = rmi_smmu_reg_read32(smmu->realm.ioaddr, SMMU_R_GERROR, &gerror); + if (ret) { + dev_err(smmu->dev, "R_SMMU: Cannot read gerror\n"); + return IRQ_HANDLED; + } + + ret = rmi_smmu_reg_read32(smmu->realm.ioaddr, SMMU_R_GERRORN, &gerrorn); + if (ret) { + dev_err(smmu->dev, "R_SMMU: Cannot read gerrorn\n"); + return IRQ_HANDLED; + } + + active = gerror ^ gerrorn; + if (!(active & GERROR_ERR_MASK)) + return IRQ_NONE; /* No errors pending */ + + dev_warn(smmu->dev, + "R_SMMU: unexpected global error reported (0x%08x), this could be serious\n", + active); + + if (active & GERROR_SFM_ERR) + dev_err(smmu->dev, "R_SMMU: device has entered Service Failure Mode!\n"); + + if (active & GERROR_MSI_GERROR_ABT_ERR) + dev_warn(smmu->dev, "R_SMMU: GERROR MSI write aborted\n"); + + if (active & GERROR_MSI_PRIQ_ABT_ERR) + dev_warn(smmu->dev, "R_SMMU: PRIQ MSI write aborted\n"); + + if (active & GERROR_MSI_EVTQ_ABT_ERR) + dev_warn(smmu->dev, "R_SMMU: EVTQ MSI write aborted\n"); + + if (active & GERROR_MSI_CMDQ_ABT_ERR) + dev_warn(smmu->dev, "R_SMMU: CMDQ MSI write aborted\n"); + + if (active & GERROR_PRIQ_ABT_ERR) + dev_err(smmu->dev, "R_SMMU: PRIQ write aborted -- events may have been lost\n"); + + if (active & GERROR_EVTQ_ABT_ERR) + dev_err(smmu->dev, "R_SMMU: EVTQ write aborted -- events may have been lost\n"); + + if (active & GERROR_CMDQ_ERR) + dev_err(smmu->dev, "R_SMMU: cmdq err\n"); + + ret = rmi_smmu_reg_write32(smmu->realm.ioaddr, SMMU_R_GERRORN, gerror); + if (ret) + dev_err(smmu->dev, "R_SMMU: Cannot write gerrorn\n"); + + return IRQ_HANDLED; +} + +static void realm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg) +{ + struct device *dev = msi_desc_to_dev(desc); + struct arm_smmu_device *smmu = dev_get_drvdata(dev); + u32 mem_attr = ARM_SMMU_MEMATTR_DEVICE_nGnRE; + phys_addr_t doorbell; + phys_addr_t *cfg; + int ret; + + doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo; + doorbell &= MSI_CFG0_ADDR_MASK; + doorbell |= MSI_CFG0_NS; + +#ifdef CONFIG_ARM_SMMU_V3_PM + /* Saves the msg (base addr of msi irq) and restores it during resume */ + desc->msg.address_lo = msg->address_lo; + desc->msg.address_hi = msg->address_hi; + desc->msg.data = msg->data; +#endif + + if (desc->msi_index != R_EVTQ_MSI_INDEX && desc->msi_index != R_GERROR_MSI_INDEX) { + dev_err(dev, "Unsupport msi_index : %u\n", desc->msi_index); + return; + } + + cfg = arm_r_smmu_msi_cfg[desc->msi_index]; + + ret = rmi_smmu_reg_write64(smmu->realm.ioaddr, cfg[0], doorbell); + if (ret) { + dev_err(dev, "Unable to write msi doorbell to %#llx\n", cfg[0]); + return; + } + + ret = rmi_smmu_reg_write32(smmu->realm.ioaddr, cfg[1], msg->data); + if (ret) { + dev_err(dev, "Unable to write msi data to %#llx\n", cfg[1]); + return; + } + + ret = rmi_smmu_reg_write32(smmu->realm.ioaddr, cfg[2], mem_attr); + if (ret) { + dev_err(dev, "Unable to write msi attr to %#llx\n", cfg[2]); + return; + } +} + +static void realm_smmu_setup_msis(struct arm_smmu_device *smmu) +{ + int ret; + struct device *dev = smmu->dev; + + ret = platform_msi_domain_alloc_range_irqs(dev, R_EVTQ_MSI_INDEX, + R_GERROR_MSI_INDEX, realm_smmu_write_msi_msg); + if (ret) { + dev_warn(dev, "R_SMMU: failed to allocate msis\n"); + return; + } + + smmu->realm.revtq.irq = msi_get_virq(dev, R_EVTQ_MSI_INDEX); + smmu->realm.rgerr_irq = msi_get_virq(dev, R_GERROR_MSI_INDEX); +} + +static int realm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu) +{ + int irq, ret; + struct realm_smmu_device *realm = &smmu->realm; + u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN; + + if (!(realm->rme_features & ARM_SMMU_FEAT_RME_MSI)) + return -EINVAL; + + /* Disable IRQs first */ + ret = realm_smmu_write_reg_sync(smmu, 0, SMMU_R_IRQ_CTRL, + SMMU_R_IRQ_CTRLACK); + if (ret) { + dev_err(smmu->dev, "failed to disable realm irqs\n"); + return ret; + } + + realm_smmu_setup_msis(smmu); + + irq = realm->revtq.irq; + if (irq) { + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + arm_r_smmu_revtq_thread, + IRQF_ONESHOT, + "arm-smmu-v3-revtq", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable evtq irq\n"); + } else { + dev_warn(smmu->dev, "no revtq irq - events will not be reported!\n"); + } + + irq = realm->rgerr_irq; + if (irq) { + ret = devm_request_threaded_irq(smmu->dev, irq, NULL, + arm_smmu_realm_gerror_handler, + IRQF_ONESHOT, + "arm-smmu-v3-rgerror", smmu); + if (ret < 0) + dev_warn(smmu->dev, "failed to enable gerror irq\n"); + } else { + dev_warn(smmu->dev, "no rgerr irq - errors will not be reported!\n"); + } + + /* Enable interrupt generation on the realm smmu */ + ret = realm_smmu_write_reg_sync(smmu, irqen_flags, SMMU_R_IRQ_CTRL, + SMMU_R_IRQ_CTRLACK); + if (ret) + dev_warn(smmu->dev, "failed to enable realm irqs\n"); + + return 0; +} + +void arm_r_smmu_device_init(struct arm_smmu_device *smmu, resource_size_t ioaddr) +{ + int ret; + u32 idr0, enables = 0; + struct realm_smmu_device *realm = &smmu->realm; + + if (is_realm_world()) + return; + + ret = rmi_smmu_reg_read32(ioaddr, SMMU_R_IDR0, &idr0); + if (ret) + return; + + realm->ioaddr = ioaddr; + realm->rme_features |= ARM_SMMU_FEAT_RME; + + if (idr0 & IDR0_ATS) + realm->rme_features |= ARM_SMMU_FEAT_ATS; + + if (idr0 & R_IDR0_MSI) + realm->rme_features |= ARM_SMMU_FEAT_RME_MSI; + + if (smmu->features & ARM_SMMU_FEAT_E2H) { + ret = rmi_smmu_reg_write32(ioaddr, SMMU_R_CR2, CR2_E2H); + if (ret) { + dev_err(smmu->dev, "failed to write realm SMMU_R_CR2\n"); + return; + } + } + + realm->sid_bitmap = devm_bitmap_zalloc(smmu->dev, 1 << smmu->sid_bits, GFP_KERNEL); + if (!realm->sid_bitmap) + return; + + realm->asid_bitmap = devm_bitmap_zalloc(smmu->dev, 1 << smmu->asid_bits, GFP_KERNEL); + if (!realm->asid_bitmap) + return; + + realm->vmid_bitmap = devm_bitmap_zalloc(smmu->dev, 1 << smmu->vmid_bits, GFP_KERNEL); + if (!realm->vmid_bitmap) + return; + + rwlock_init(&realm->fwd_lock); + + realm->rcmdq.max_n_shift = smmu->cmdq.q.llq.max_n_shift; + /* realm cmdq */ + ret = realm_smmu_init_queue(smmu, &realm->rcmdq, CMDQ_ENT_DWORDS, + SMMU_R_CMDQ, "rcmdq"); + if (ret) + goto err_remove_device; + + /* If SMMU support ATS, SMMU-R should enable ATSCHK */ + if (realm->rme_features & ARM_SMMU_FEAT_ATS) + enables |= CR0_ATSCHK; + + enables |= CR0_CMDQEN; + ret = realm_smmu_write_reg_sync(smmu, enables, SMMU_R_CR0, SMMU_R_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable realm command queue\n"); + goto err_remove_device; + } + + realm->revtq.max_n_shift = smmu->evtq.q.llq.max_n_shift; + /* realm evtq */ + ret = realm_smmu_init_queue(smmu, &realm->revtq, EVTQ_ENT_DWORDS, + SMMU_R_EVTQ, "revtq"); + if (ret) + goto err_remove_device; + + enables |= CR0_EVTQEN; + ret = realm_smmu_write_reg_sync(smmu, enables, SMMU_R_CR0, SMMU_R_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable realm event queue\n"); + goto err_remove_device; + } + + ret = realm_smmu_init_strtab(smmu); + if (ret) + goto err_remove_device; + + ret = realm_smmu_setup_unique_irqs(smmu); + if (ret) { + dev_err(smmu->dev, "RME failed to setup irqs\n"); + goto err_remove_device; + } + + if (is_kdump_kernel()) + enables &= ~CR0_EVTQEN; + + /* Enable the SMMU interface */ + enables |= CR0_SMMUEN; + + ret = realm_smmu_write_reg_sync(smmu, enables, SMMU_R_CR0, SMMU_R_CR0ACK); + if (ret) { + dev_err(smmu->dev, "failed to enable realm smmu interface\n"); + goto err_remove_device; + } + + realm->enabled = true; + + return; + +err_remove_device: + arm_r_smmu_device_remove(smmu); +} + +static void arm_r_smmu_free_l2_strtab(struct arm_smmu_device *smmu, + struct realm_smmu_strtab_cfg *cfg) +{ + size_t size = (1 << STRTAB_SPLIT) * sizeof(struct arm_smmu_ste); + struct arm_smmu_strtab_l1_desc *desc; + unsigned int i; + + if (!cfg->l1_desc) + return; + + for (i = 0; i < cfg->num_l1_ents; i++) { + desc = &cfg->l1_desc[i]; + if (!desc->l2ptr) + continue; + realm_config_strtab_l2(SMMU_STRTAB_L2_DEINIT, + smmu->realm.ioaddr, cfg->strtab_dma, + i, desc->l2ptr_dma, desc->span); + if (WARN_ON(granule_undelegate_range(desc->l2ptr_dma, size))) + continue; + dma_free_coherent(smmu->dev, size, desc->l2ptr, desc->l2ptr_dma); + } + kfree(cfg->l1_desc); +} + +void arm_r_smmu_device_remove(struct arm_smmu_device *smmu) +{ + + size_t l1size, qsz; + struct realm_smmu_queue *q; + unsigned long ioaddr = smmu->realm.ioaddr; + struct realm_smmu_device *realm = &smmu->realm; + struct realm_smmu_strtab_cfg *cfg = &realm->strtab_cfg; + + if (is_realm_world()) + return; + + if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) { + l1size = cfg->num_l1_ents * (STRTAB_L1_DESC_DWORDS << 3); + arm_r_smmu_free_l2_strtab(smmu, cfg); + } else { + l1size = cfg->num_l1_ents * sizeof(cfg->strtab.linear[0]); + } + + if (cfg->strtab.linear) { + realm_config_strtab(SMMU_STRTAB_DEINIT, ioaddr, cfg->strtab_dma, + cfg->strtab_base_cfg); + if (!WARN_ON(granule_undelegate_range(cfg->strtab_dma, l1size))) + dma_free_coherent(smmu->dev, l1size, cfg->strtab.linear, + cfg->strtab_dma); + } + + if (realm->rcmdq.base) { + q = &realm->rcmdq; + realm_config_queue(SMMU_QUEUE_DEINIT, ioaddr, SMMU_R_CMDQ, + q->base_dma, q->max_n_shift); + qsz = ((1 << q->max_n_shift) * CMDQ_ENT_DWORDS) << 3; + if (!WARN_ON(granule_undelegate_range(q->base_dma, qsz))) + dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma); + } + + if (realm->revtq.base) { + q = &realm->revtq; + realm_config_queue(SMMU_QUEUE_DEINIT, ioaddr, SMMU_R_EVTQ, + q->base_dma, q->max_n_shift); + qsz = ((1 << q->max_n_shift) * EVTQ_ENT_DWORDS) << 3; + if (!WARN_ON(granule_undelegate_range(q->base_dma, qsz))) + dma_free_coherent(smmu->dev, qsz, q->base, q->base_dma); + } +} diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.h new file mode 100644 index 000000000000..123b4a2f8063 --- /dev/null +++ b/drivers/iommu/arm/arm-smmu-v3/arm-r-smmu-v3.h @@ -0,0 +1,83 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* + * Copyright (c) 2024, The Linux Foundation. All rights reserved. + */ + +#ifndef _ARM_R_SMMU_V3_H +#define _ARM_R_SMMU_V3_H + +#include + +#define MSI_CFG0_NS (1UL << 63) + +#define SMMU_R_IDR0 0 +#define R_IDR0_MSI (1U << 13) +#define SMMU_R_GERROR 0x60 +#define SMMU_R_GERRORN 0x64 + +#define SMMU_R_CMDQ 0 +#define SMMU_R_EVTQ 1 + +#define SMMU_R_CR0 0x20 +#define SMMU_R_CR0ACK 0x24 + +#define SMMU_R_CR2 0x2C + +#define SMMU_R_IRQ_CTRL 0x50 +#define SMMU_R_IRQ_CTRLACK 0x54 + +#define SMMU_R_GERROR_IRQ_CFG0 0x68 +#define SMMU_R_GERROR_IRQ_CFG1 0x70 +#define SMMU_R_GERROR_IRQ_CFG2 0x74 + +#define SMMU_R_EVENTQ_IRQ_CFG0 0xb0 +#define SMMU_R_EVENTQ_IRQ_CFG1 0xb8 +#define SMMU_R_EVENTQ_IRQ_CFG2 0xbc + +#define STRTAB_L1_DESC_DWORDS 1 + +#define CTXDESC_SPLIT 10 +#define CTXDESC_L1_DESC_DWORDS 1 + +#define realm_read_poll_timeout(op, val, cond, delay_us, timeout_us, \ + delay_before_read, args...) \ +({ \ + u64 __timeout_us = (timeout_us); \ + int __ret = 0; \ + unsigned long __delay_us = (delay_us); \ + ktime_t __timeout = ktime_add_us(ktime_get(), __timeout_us); \ + if (delay_before_read && __delay_us) \ + udelay(__delay_us); \ + for (;;) { \ + __ret = op(args, &val); \ + if (__ret) \ + break; \ + if (cond) \ + break; \ + if (__timeout_us && \ + ktime_compare(ktime_get(), __timeout) > 0) { \ + __ret = op(args, &val); \ + break; \ + } \ + if (__delay_us) \ + udelay(__delay_us); \ + cpu_relax(); \ + } \ + __ret ? __ret : ((cond) ? 0 : -ETIMEDOUT); \ +}) + +bool arm_smmu_support_rme(struct arm_smmu_device *smmu); +void realm_smmu_write_cd(struct arm_smmu_master *master, int ssid, + const struct arm_smmu_cd *target); +void realm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, + const struct arm_smmu_ste *target); +void realm_smmu_free_cd_tables(struct arm_smmu_master *master); +int realm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, + u64 *cmds, int n, bool sync); +void arm_r_smmu_device_init(struct arm_smmu_device *smmu, resource_size_t ioaddr); +void arm_r_smmu_device_remove(struct arm_smmu_device *smmu); + +void realm_smmu_attach_dev(struct arm_smmu_domain *smmu_domain, struct arm_smmu_master *master); +void realm_smmu_domain_clear(struct arm_smmu_domain *smmu_domain); + +#endif /* _ARM_R_SMMU_V3_H */ diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c index 06995ee563d5..5f1272a93a1c 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-iommufd.c @@ -7,6 +7,11 @@ #include "arm-smmu-v3.h" +#ifdef CONFIG_HISI_CCA_DA +#include +#include "arm-r-smmu-v3.h" +#endif + void *arm_smmu_hw_info(struct device *dev, u32 *length, u32 *type) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); @@ -129,6 +134,11 @@ static int arm_smmu_attach_dev_nested(struct iommu_domain *domain, return ret; } +#ifdef CONFIG_HISI_CCA_DA + if (nested_domain->vsmmu->s2_parent->realm) + realm_smmu_attach_dev(nested_domain->vsmmu->s2_parent, master); +#endif + arm_smmu_make_nested_domain_ste(&ste, master, nested_domain, state.ats_enabled); arm_smmu_install_ste_for_dev(master, &ste); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c index 9342fac71801..a5070c1a8bdd 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3-sva.c @@ -255,6 +255,11 @@ bool arm_smmu_master_iopf_supported(struct arm_smmu_master *master) if (master->num_streams != 1) return false; +#ifdef CONFIG_HISI_CCA_DA + if (is_realm_world()) + return true; +#endif + return master->stall_enabled; } diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c index 1569090b1b12..4fbb1f739847 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.c @@ -37,6 +37,11 @@ #include "arm-s-smmu-v3.h" #endif +#ifdef CONFIG_HISI_CCA_DA +#include +#include "arm-r-smmu-v3.h" +#endif + static bool disable_msipolling; module_param(disable_msipolling, bool, 0444); MODULE_PARM_DESC(disable_msipolling, @@ -965,6 +970,14 @@ int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu, struct arm_smmu_ll_queue llq, head; int ret = 0; +#ifdef CONFIG_HISI_CCA_DA + if (smmu->realm.forward_cmd) { + ret = realm_smmu_cmdq_issue_cmdlist(smmu, cmds, n, sync); + if (ret) + return ret; + } +#endif + #ifdef CONFIG_ARM_SMMU_V3_ECMDQ if (!cmdq->shared) return arm_smmu_ecmdq_issue_cmdlist(smmu, cmdq, cmds, n, sync); @@ -1592,6 +1605,9 @@ void arm_smmu_make_s1_cd(struct arm_smmu_cd *target, target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr & CTXDESC_CD_1_TTB0_MASK); target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair); +#ifdef CONFIG_HISI_CCA_DA + master->ns_ttbr = pgtbl_cfg->ns_ttbr; +#endif } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd); @@ -1739,6 +1755,10 @@ static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid, .sid = sid, }; +#ifdef CONFIG_HISI_CCA_DA + realm_smmu_write_ste(master, sid, target); +#endif + arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data); /* It's likely that we'll want to use the new STE soon */ @@ -1882,6 +1902,9 @@ void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target, target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr & STRTAB_STE_3_S2TTB_MASK); +#ifdef CONFIG_HISI_CCA_DA + master->ns_ttbr = pgtbl_cfg->ns_ttbr; +#endif } EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste); @@ -2611,6 +2634,11 @@ static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev) struct arm_smmu_master *master = dev_iommu_priv_get(dev); int ret; +#ifdef CONFIG_HISI_CCA_DA + if (dev_is_pci(dev) && is_realm_pcipc_ns(dev)) + smmu_domain->pcipc_ns_dev = true; +#endif + ret = arm_smmu_domain_finalise(smmu_domain, master->smmu, 0); if (ret) { kfree(smmu_domain); @@ -2627,6 +2655,10 @@ static void arm_smmu_domain_free_paging(struct iommu_domain *domain) free_io_pgtable_ops(smmu_domain->pgtbl_ops); +#ifdef CONFIG_HISI_CCA_DA + realm_smmu_domain_clear(smmu_domain); +#endif + /* Free the ASID or VMID */ if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) { /* Prevent SVA from touching the CD while we're freeing it */ @@ -2731,6 +2763,20 @@ static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain, return -EINVAL; } +#ifdef CONFIG_HISI_CCA_DA + if (smmu_domain->realm) { + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + fmt = CCA_REALM_S1; + else + fmt = CCA_REALM_S2; + } else if (smmu_domain->pcipc_ns_dev) { + if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) + fmt = CCA_REALM_NS_S1; + else + fmt = CCA_REALM_NS_S2; + } +#endif + if (smmu->features & ARM_SMMU_FEAT_HD) pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD; @@ -3135,6 +3181,10 @@ static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev) return ret; } +#ifdef CONFIG_HISI_CCA_DA + realm_smmu_attach_dev(smmu_domain, master); +#endif + switch (smmu_domain->stage) { case ARM_SMMU_DOMAIN_S1: { struct arm_smmu_cd target_cd; @@ -3389,8 +3439,14 @@ arm_smmu_domain_alloc_user(struct device *dev, u32 flags, const struct iommu_user_data *user_data) { struct arm_smmu_master *master = dev_iommu_priv_get(dev); +#ifdef CONFIG_HISI_CCA_DA + const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING | + IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_RME; +#else const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING | IOMMU_HWPT_ALLOC_NEST_PARENT; +#endif + struct arm_smmu_domain *smmu_domain; int ret; @@ -3412,6 +3468,11 @@ arm_smmu_domain_alloc_user(struct device *dev, u32 flags, smmu_domain->nest_parent = true; } +#ifdef CONFIG_HISI_CCA_DA + if (flags & IOMMU_HWPT_RME) + smmu_domain->realm = true; +#endif + smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED; smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops; ret = arm_smmu_domain_finalise(smmu_domain, master->smmu, flags); @@ -3644,6 +3705,12 @@ static struct iommu_device *arm_smmu_probe_device(struct device *dev) smmu->features & ARM_SMMU_FEAT_STALL_FORCE) master->stall_enabled = true; +#ifdef CONFIG_HISI_CCA_DA + /* Realm don't support STALL_MODEL */ + if (is_realm_world()) + master->stall_enabled = false; +#endif + return &smmu->iommu; err_free_master: @@ -3668,6 +3735,9 @@ static void arm_smmu_release_device(struct device *dev) arm_smmu_remove_master(master); if (arm_smmu_cdtab_allocated(&master->cd_table)) arm_smmu_free_cd_tables(master); +#ifdef CONFIG_HISI_CCA_DA + realm_smmu_free_cd_tables(master); +#endif kfree(master); } @@ -3912,6 +3982,24 @@ static int arm_smmu_clear_dirty_log(struct iommu_domain *domain, } #endif +#ifdef CONFIG_HISI_CCA_DA +static int arm_smmu_enable_rme(struct iommu_domain *domain) +{ + struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); + struct arm_smmu_device *smmu = smmu_domain->smmu; + int ret = 0; + + mutex_lock(&smmu_domain->init_mutex); + if (arm_smmu_support_rme(smmu)) + smmu_domain->realm = true; + else + ret = -EOPNOTSUPP; + mutex_unlock(&smmu_domain->init_mutex); + + return ret; +} +#endif + static int arm_smmu_of_xlate(struct device *dev, struct of_phandle_args *args) { return iommu_fwspec_add_ids(dev, args->args, 1); @@ -4158,6 +4246,9 @@ static struct iommu_ops arm_smmu_ops = { .iotlb_sync_map = arm_smmu_iotlb_sync_map, #endif .iova_to_phys = arm_smmu_iova_to_phys, +#ifdef CONFIG_HISI_CCA_DA + .enable_rme = arm_smmu_enable_rme, +#endif #ifdef CONFIG_ARM_SMMU_V3_HTTU .support_dirty_log = arm_smmu_support_dirty_log, .switch_dirty_log = arm_smmu_switch_dirty_log, @@ -5707,6 +5798,10 @@ static int arm_smmu_device_probe(struct platform_device *pdev) virtcca_smmu_device_init(pdev, smmu, ioaddr, false); #endif +#ifdef CONFIG_HISI_CCA_DA + arm_r_smmu_device_init(smmu, ioaddr); +#endif + /* And we're up. Go go go! */ ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL, "smmu3.%pa", &ioaddr); @@ -5734,6 +5829,10 @@ static void arm_smmu_device_remove(struct platform_device *pdev) { struct arm_smmu_device *smmu = platform_get_drvdata(pdev); +#ifdef CONFIG_HISI_CCA_DA + arm_r_smmu_device_remove(smmu); +#endif + iommu_device_unregister(&smmu->iommu); iommu_device_sysfs_remove(&smmu->iommu); arm_smmu_device_disable(smmu); diff --git a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h index a4847edf3ec8..9128f8286a69 100644 --- a/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h +++ b/drivers/iommu/arm/arm-smmu-v3/arm-smmu-v3.h @@ -751,6 +751,77 @@ struct arm_smmu_strtab_cfg { }; }; +#ifdef CONFIG_HISI_CCA_DA +struct realm_smmu_queue { + __le64 *base; + dma_addr_t base_dma; + u32 max_n_shift; + int irq; /* Wired interrupt */ +}; + +struct arm_smmu_l1_ctx_desc { + struct arm_smmu_cd *l2ptr; + dma_addr_t l2ptr_dma; +}; + +struct realm_smmu_ctx_desc_cfg { + union { + struct arm_smmu_cd *linear; + __le64 *l1_desc; + } cdtab; + dma_addr_t cdtab_dma; + struct arm_smmu_l1_ctx_desc *l1_desc; + unsigned int num_l1_ents; + unsigned int used_ssids; + u8 used_sid; + u8 in_ste; + u8 s1fmt; + /* log2 of the maximum number of CDs supported by this table */ + u8 s1cdmax; +}; + +/* High-level stream table and context descriptor structures */ +struct arm_smmu_strtab_l1_desc { + u8 span; + + struct arm_smmu_ste *l2ptr; + dma_addr_t l2ptr_dma; +}; + +struct realm_smmu_strtab_cfg { + union { + struct arm_smmu_ste *linear; + __le64 *l1_desc; + } strtab; + dma_addr_t strtab_dma; + struct arm_smmu_strtab_l1_desc *l1_desc; + unsigned int num_l1_ents; + + u64 strtab_base; + u32 strtab_base_cfg; +}; + +/* An SMMUv3 realm instance */ +struct realm_smmu_device { + resource_size_t ioaddr; + struct realm_smmu_strtab_cfg strtab_cfg; + struct realm_smmu_queue rcmdq; + struct realm_smmu_queue revtq; + int rgerr_irq; + + unsigned long *sid_bitmap; + unsigned long *asid_bitmap; + unsigned long *vmid_bitmap; + rwlock_t fwd_lock; + +#define ARM_SMMU_FEAT_RME (1 << 0) +#define ARM_SMMU_FEAT_RME_MSI (1 << 1) + u32 rme_features; + bool forward_cmd; + bool enabled; +}; +#endif + /* An SMMUv3 instance */ struct arm_smmu_device { struct device *dev; @@ -840,6 +911,10 @@ struct arm_smmu_device { resource_size_t ioaddr; uint64_t s_smmu_id; #endif + +#ifdef CONFIG_HISI_CCA_DA + struct realm_smmu_device realm; +#endif }; struct arm_smmu_stream { @@ -861,6 +936,12 @@ struct arm_smmu_master { bool stall_enabled; bool sva_enabled; bool iopf_enabled; +#ifdef CONFIG_HISI_CCA_DA + bool realm; + bool pcipc_ns_dev; + struct realm_smmu_ctx_desc_cfg realm_cd_table; + unsigned long ns_ttbr; +#endif unsigned int ssid_bits; u16 partid; u8 pmg; @@ -899,6 +980,10 @@ struct arm_smmu_domain { struct list_head node; struct kvm *kvm; #endif +#ifdef CONFIG_HISI_CCA_DA + bool realm; + bool pcipc_ns_dev; +#endif }; struct arm_smmu_nested_domain { diff --git a/drivers/iommu/io-pgtable-realm.c b/drivers/iommu/io-pgtable-realm.c new file mode 100644 index 000000000000..8b34b0858a5a --- /dev/null +++ b/drivers/iommu/io-pgtable-realm.c @@ -0,0 +1,1019 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * CCA Realm page table allocator. + * + * Copyright (c) 2024 Hisilicon Limited. + * + * Based on io-pgtable-arm. + * + * Copyright (C) 2014 ARM Limited + * + * Author: Will Deacon + */ + +#define pr_fmt(fmt) "realm io-pgtable: " fmt + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "io-pgtable-arm.h" + +#define REALM_MAX_ADDR_BITS 48 +#define REALM_S2_MAX_CONCAT_PAGES 16 +#define REALM_MAX_LEVELS 4 + +/* Struct accessors */ +#define io_pgtable_to_data(x) \ + container_of((x), struct realm_io_pgtable, iop) + +#define io_pgtable_ops_to_data(x) \ + io_pgtable_to_data(io_pgtable_ops_to_pgtable(x)) + +/* + * Calculate the right shift amount to get to the portion describing level l + * in a virtual address mapped by the pagetable in d. + */ +#define REALM_LVL_SHIFT(l, d) \ + (((REALM_MAX_LEVELS - (l)) * (d)->bits_per_level) + \ + ilog2(sizeof(realm_iopte))) + +#define REALM_GRANULE(d) \ + (sizeof(realm_iopte) << (d)->bits_per_level) +#define REALM_PGD_SIZE(d) \ + (sizeof(realm_iopte) << (d)->pgd_bits) + +#define REALM_PTES_PER_TABLE(d) \ + (REALM_GRANULE(d) >> ilog2(sizeof(realm_iopte))) + +/* + * Calculate the index at level l used to map virtual address a using the + * pagetable in d. + */ +#define REALM_PGD_IDX(l, d) \ + ((l) == (d)->start_level ? (d)->pgd_bits - (d)->bits_per_level : 0) + +#define REALM_LVL_IDX(a, l, d) \ + (((u64)(a) >> REALM_LVL_SHIFT(l, d)) & \ + ((1 << ((d)->bits_per_level + REALM_PGD_IDX(l, d))) - 1)) + +/* Calculate the block/page mapping size at level l for pagetable in d. */ +#define REALM_BLOCK_SIZE(l, d) (1ULL << REALM_LVL_SHIFT(l, d)) + +/* Page table bits */ +#define REALM_PTE_TYPE_SHIFT 0 +#define REALM_PTE_TYPE_MASK 0x3 + +#define REALM_PTE_TYPE_BLOCK 1 +#define REALM_PTE_TYPE_TABLE 3 +#define REALM_PTE_TYPE_PAGE 3 + +#define REALM_PTE_ADDR_MASK GENMASK_ULL(47, 12) + +#define REALM_PTE_XN (((realm_iopte)3) << 53) +#define REALM_PTE_AF (((realm_iopte)1) << 10) +#define REALM_PTE_SH_IS (((realm_iopte)3) << 8) +#define REALM_PTE_NS (((realm_iopte)1) << 5) +#define REALM_PTE_VALID (((realm_iopte)1) << 0) + +/* Software bit for solving coherency races */ +#define REALM_PTE_SW_SYNC (((realm_iopte)1) << 55) + +/* Stage-1 PTE */ +#define REALM_PTE_AP_UNPRIV (((realm_iopte)1) << 6) +#define REALM_PTE_AP_RDONLY (((realm_iopte)2) << 6) +#define REALM_PTE_ATTRINDX_SHIFT 2 +#define REALM_PTE_nG (((realm_iopte)1) << 11) + +/* Stage-2 PTE */ +#define REALM_PTE_HAP_FAULT (((realm_iopte)0) << 6) +#define REALM_PTE_HAP_READ (((realm_iopte)1) << 6) +#define REALM_PTE_HAP_WRITE (((realm_iopte)2) << 6) +#define REALM_PTE_MEMATTR_OIWB (((realm_iopte)0xf) << 2) +#define REALM_PTE_MEMATTR_NC (((realm_iopte)0x5) << 2) +#define REALM_PTE_MEMATTR_DEV (((realm_iopte)0x1) << 2) + +/* Register bits */ +#define REALM_VTCR_SL0_MASK 0x3 + +#define REALM_MAIR_ATTR_SHIFT(n) ((n) << 3) +#define REALM_MAIR_ATTR_MASK 0xff +#define REALM_MAIR_ATTR_DEVICE 0x04 +#define REALM_MAIR_ATTR_NC 0x44 +#define REALM_MAIR_ATTR_INC_OWBRWA 0xf4 +#define REALM_MAIR_ATTR_WBRWA 0xff +#define REALM_MAIR_ATTR_IDX_NC 0 +#define REALM_MAIR_ATTR_IDX_CACHE 1 +#define REALM_MAIR_ATTR_IDX_DEV 2 +#define REALM_MAIR_ATTR_IDX_INC_OCACHE 3 + +/* IOPTE accessors */ +#define iopte_deref(pte, d) __va(iopte_to_paddr(pte, d)) + +#define iopte_type(pte) \ + (((pte) >> REALM_PTE_TYPE_SHIFT) & REALM_PTE_TYPE_MASK) + +struct realm_io_pgtable { + struct io_pgtable iop; + int pgd_bits; + int start_level; + int bits_per_level; + void *pgd; + bool ns; + void *ns_pgd; +}; + +typedef u64 realm_iopte; + +static inline bool iopte_leaf(realm_iopte pte, int lvl, + enum io_pgtable_fmt fmt) +{ + if (lvl == (REALM_MAX_LEVELS - 1)) + return iopte_type(pte) == REALM_PTE_TYPE_PAGE; + + return iopte_type(pte) == REALM_PTE_TYPE_BLOCK; +} + +static phys_addr_t iopte_to_paddr(realm_iopte pte, + struct realm_io_pgtable *data) +{ + u64 paddr = pte & REALM_PTE_ADDR_MASK; + + if (REALM_GRANULE(data) < SZ_64K) + return paddr; + + /* Rotate the packed high-order bits back to the top */ + return (paddr | (paddr << (48 - 12))) & (REALM_PTE_ADDR_MASK << 4); +} + +static realm_iopte paddr_to_iopte(phys_addr_t paddr, + struct realm_io_pgtable *data) +{ + realm_iopte pte = paddr; + + /* Of the bits which overlap, either 51:48 or 15:12 are always RES0 */ + return (pte | (pte >> (48 - 12))) & REALM_PTE_ADDR_MASK; +} + +static void *__realm_alloc_pages(size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg) +{ + struct device *dev = cfg->iommu_dev; + int order = get_order(size); + struct page *p; + void *pages; + + p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); + if (unlikely(!p)) + return NULL; + + pages = page_address(p); + if (granule_delegate_range(virt_to_phys(pages), size)) { + free_pages((unsigned long)pages, order); + return NULL; + } + + return pages; +} + +static void __realm_free_pages(void *pages, size_t size, + struct io_pgtable_cfg *cfg) +{ + /* If the undelegate fails then leak the pages */ + if (WARN_ON(granule_undelegate_range(virt_to_phys(pages), size))) + return; + + free_pages((unsigned long)pages, get_order(size)); +} + +static realm_iopte realm_prot_to_pte(struct realm_io_pgtable *data, int prot) +{ + realm_iopte pte; + + if (data->iop.fmt == CCA_REALM_S1 || data->iop.fmt == CCA_REALM_NS_S1) { + pte = REALM_PTE_nG; + if (!(prot & IOMMU_WRITE) && (prot & IOMMU_READ)) + pte |= REALM_PTE_AP_RDONLY; + if (!(prot & IOMMU_PRIV)) + pte |= REALM_PTE_AP_UNPRIV; + + if (prot & IOMMU_MMIO) + pte |= (REALM_MAIR_ATTR_IDX_DEV + << REALM_PTE_ATTRINDX_SHIFT); + else if (prot & IOMMU_CACHE) + pte |= (REALM_MAIR_ATTR_IDX_CACHE + << REALM_PTE_ATTRINDX_SHIFT); + } else { + pte = REALM_PTE_HAP_FAULT; + if (prot & IOMMU_READ) + pte |= REALM_PTE_HAP_READ; + if (prot & IOMMU_WRITE) + pte |= REALM_PTE_HAP_WRITE; + + if (prot & IOMMU_MMIO) + pte |= REALM_PTE_MEMATTR_DEV; + else if (prot & IOMMU_CACHE) + pte |= REALM_PTE_MEMATTR_OIWB; + else + pte |= REALM_PTE_MEMATTR_NC; + } + + if (prot & IOMMU_CACHE) + pte |= REALM_PTE_SH_IS; + + if (prot & IOMMU_NOEXEC) + pte |= REALM_PTE_XN; + + pte |= REALM_PTE_AF; + + return pte; +} + +static int realm_map(struct realm_io_pgtable *data, unsigned long iova, + phys_addr_t paddr, size_t size, size_t pgcount, + realm_iopte prot, int lvl, realm_iopte *pgd, + gfp_t gfp, size_t *mapped) +{ + int ret = 0; + realm_iopte *cptep; + phys_addr_t phys, tbl_entry; + size_t page_attr, block_size; + size_t tblsz = REALM_GRANULE(data); + unsigned long pgdp = virt_to_phys(pgd), map_cnt; + + if (size > SZ_1G || pgcount > U32_MAX) + return -EINVAL; + + page_attr = size | (pgcount << 32); + + lvl = rmi_smmu_map(pgdp, iova, paddr, prot, page_attr, &map_cnt); + if (lvl == REALM_MAX_LEVELS) + *mapped += map_cnt * size; + if (lvl < 0) + return -EINVAL; + + for (; lvl < REALM_MAX_LEVELS; lvl++) { + block_size = REALM_BLOCK_SIZE(lvl, data); + + /* If we can install a leaf entry at this level, then do so */ + if (size == block_size) { + lvl = rmi_smmu_map(pgdp, iova, paddr, prot, page_attr, &map_cnt); + if (lvl == REALM_MAX_LEVELS) + *mapped += map_cnt * size; + break; + } + + /* We can't allocate tables at the final level */ + if (WARN_ON(lvl >= REALM_MAX_LEVELS - 1)) + return -EINVAL; + + cptep = (realm_iopte *)__realm_alloc_pages(tblsz, GFP_KERNEL, + &data->iop.cfg); + if (!cptep) + return -ENOMEM; + + phys = virt_to_phys(cptep); + tbl_entry = phys | REALM_PTE_TYPE_TABLE; + ret = rmi_smmu_page_table_create(pgdp, iova, tbl_entry, tblsz, + lvl); + if (ret) { + __realm_free_pages(cptep, tblsz, &data->iop.cfg); + break; + } + } + + return ret; +} + +static void __realm_free_pgtable(struct realm_io_pgtable *data, int lvl, + void *va, unsigned long iova) +{ + unsigned long i, entry_num, pte, table_size, next, pa; + + if (lvl == data->start_level) + table_size = REALM_PGD_SIZE(data); + else + table_size = REALM_GRANULE(data); + + entry_num = table_size / sizeof(realm_iopte); + for (i = 0; i < entry_num; i++) { + next = iova + (i << REALM_LVL_SHIFT(lvl, data)); + rmi_smmu_read_pte(virt_to_phys(data->pgd), next, lvl, &pte); + if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) + continue; + + pa = pte & REALM_PTE_ADDR_MASK; + __realm_free_pgtable(data, lvl + 1, __va(pa), next); + } + (void)rmi_smmu_page_table_destroy(virt_to_phys(data->pgd), iova, + virt_to_phys(va), table_size, lvl); + __realm_free_pages(va, table_size, &data->iop.cfg); +} + +static size_t realm_unmap(struct realm_io_pgtable *data, + struct iommu_iotlb_gather *gather, + unsigned long iova, size_t size, size_t pgcount, + int lvl, realm_iopte *pgd) +{ + unsigned long map_cnt; + + if (size > SZ_1G || pgcount > U32_MAX) + return -EINVAL; + + (void)rmi_smmu_map(virt_to_phys(pgd), iova, 0, 0, size | (pgcount << 32), + &map_cnt); + + return map_cnt * size; +} + +static phys_addr_t __realm_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct realm_io_pgtable *data = io_pgtable_ops_to_data(ops); + unsigned long phys; + int lvl; + + lvl = rmi_smmu_iova_to_phys(virt_to_phys(data->pgd), iova, &phys); + if (!phys) + return 0; + + iova &= (REALM_BLOCK_SIZE(lvl, data) - 1); + return (phys_addr_t)phys | iova; +} + +static void *realm_ns_alloc_pages(size_t size, gfp_t gfp, + struct io_pgtable_cfg *cfg, + void *cookie) +{ + struct device *dev = cfg->iommu_dev; + int order = get_order(size); + void *pages; + + VM_BUG_ON((gfp & __GFP_HIGHMEM)); + + if (cfg->alloc) { + pages = cfg->alloc(cookie, size, gfp); + } else { + struct page *p; + + p = alloc_pages_node(dev_to_node(dev), gfp | __GFP_ZERO, order); + pages = p ? page_address(p) : NULL; + } + + if (!pages) + return NULL; + + return pages; +} + +static void realm_ns_free_pages(void *pages, size_t size, + struct io_pgtable_cfg *cfg, + void *cookie) +{ + if (cfg->free) + cfg->free(cookie, pages, size); + else + free_pages((unsigned long)pages, get_order(size)); +} + +static void realm_ns_init_pte(struct realm_io_pgtable *data, + phys_addr_t paddr, realm_iopte prot, + int lvl, int num_entries, realm_iopte *ptep) +{ + realm_iopte pte = prot; + size_t sz = REALM_BLOCK_SIZE(lvl, data); + int i; + + if (data->iop.fmt != ARM_MALI_LPAE && lvl == REALM_MAX_LEVELS - 1) + pte |= REALM_PTE_TYPE_PAGE; + else + pte |= REALM_PTE_TYPE_BLOCK; + + for (i = 0; i < num_entries; i++) + ptep[i] = pte | paddr_to_iopte(paddr + i * sz, data); +} + +static size_t realm_ns_unmap(struct realm_io_pgtable *data, + struct iommu_iotlb_gather *gather, + unsigned long iova, size_t size, size_t pgcount, + int lvl, realm_iopte *ptep); + +static int realm_init_pte(struct realm_io_pgtable *data, + unsigned long iova, phys_addr_t paddr, + realm_iopte prot, int lvl, int num_entries, + realm_iopte *ptep) +{ + int i; + + for (i = 0; i < num_entries; i++) + if (iopte_leaf(ptep[i], lvl, data->iop.fmt)) { + /* We require an unmap first */ + return -EEXIST; + } else if (iopte_type(ptep[i]) == REALM_PTE_TYPE_TABLE) { + /* + * We need to unmap and free the old table before + * overwriting it with a block entry. + */ + realm_iopte *tblp; + size_t sz = REALM_BLOCK_SIZE(lvl, data); + + tblp = ptep - REALM_LVL_IDX(iova, lvl, data); + if (realm_ns_unmap(data, NULL, iova + i * sz, sz, 1, + lvl, tblp) != sz) { + WARN_ON(1); + return -EINVAL; + } + } + + realm_ns_init_pte(data, paddr, prot, lvl, num_entries, ptep); + return 0; +} + +static realm_iopte realm_install_table(realm_iopte *table, realm_iopte *ptep, + realm_iopte curr, + struct realm_io_pgtable *data) +{ + realm_iopte old, new; + + new = paddr_to_iopte(__pa(table), data) | REALM_PTE_TYPE_TABLE; + /* + * Ensure the table itself is visible before its PTE can be. + * Whilst we could get away with cmpxchg64_release below, this + * doesn't have any ordering semantics when !CONFIG_SMP. + */ + dma_wmb(); + + old = cmpxchg64_relaxed(ptep, curr, new); + + return old; +} + +static int realm_ns_map(struct realm_io_pgtable *data, unsigned long iova, + phys_addr_t paddr, size_t size, size_t pgcount, + realm_iopte prot, int lvl, realm_iopte *ptep, + gfp_t gfp, size_t *mapped) +{ + realm_iopte *cptep, pte; + size_t block_size = REALM_BLOCK_SIZE(lvl, data); + size_t tblsz = REALM_GRANULE(data); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + int ret = 0, num_entries, max_entries, map_idx_start; + + /* Find our entry at the current level */ + map_idx_start = REALM_LVL_IDX(iova, lvl, data); + ptep += map_idx_start; + + /* If we can install a leaf entry at this level, then do so */ + if (size == block_size) { + max_entries = REALM_PTES_PER_TABLE(data) - map_idx_start; + num_entries = min_t(int, pgcount, max_entries); + ret = realm_init_pte(data, iova, paddr, prot, lvl, num_entries, ptep); + if (!ret) + *mapped += num_entries * size; + + return ret; + } + + /* We can't allocate tables at the final level */ + if (WARN_ON(lvl >= REALM_MAX_LEVELS - 1)) + return -EINVAL; + + /* Grab a pointer to the next level */ + pte = READ_ONCE(*ptep); + if (!pte) { + cptep = realm_ns_alloc_pages(tblsz, gfp, cfg, data->iop.cookie); + if (!cptep) + return -ENOMEM; + + pte = realm_install_table(cptep, ptep, 0, data); + if (pte) + realm_ns_free_pages(cptep, tblsz, cfg, data->iop.cookie); + +#ifdef CONFIG_HISILICON_ERRATUM_162100602 + if (lvl <= 2) + io_pgtable_tlb_flush_walk(&data->iop, iova, 0, REALM_GRANULE(data)); +#endif + } + + if (pte && !iopte_leaf(pte, lvl, data->iop.fmt)) { + cptep = iopte_deref(pte, data); + } else if (pte) { + /* We require an unmap first */ + return -EEXIST; + } + + /* Rinse, repeat */ + return realm_ns_map(data, iova, paddr, size, pgcount, prot, lvl + 1, + cptep, gfp, mapped); +} + +static void __realm_ns_free_pgtable(struct realm_io_pgtable *data, int lvl, + realm_iopte *ptep) +{ + realm_iopte *start, *end; + unsigned long table_size; + + if (lvl == data->start_level) + table_size = REALM_PGD_SIZE(data); + else + table_size = REALM_GRANULE(data); + + start = ptep; + + /* Only leaf entries at the last level */ + if (lvl == REALM_MAX_LEVELS - 1) + end = ptep; + else + end = (void *)ptep + table_size; + + while (ptep != end) { + realm_iopte pte = *ptep++; + + if (!pte || iopte_leaf(pte, lvl, data->iop.fmt)) + continue; + + __realm_ns_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); + } + + realm_ns_free_pages(start, table_size, &data->iop.cfg, data->iop.cookie); +} + +static size_t realm_ns_unmap(struct realm_io_pgtable *data, + struct iommu_iotlb_gather *gather, + unsigned long iova, size_t size, size_t pgcount, + int lvl, realm_iopte *ptep) +{ + realm_iopte pte; + struct io_pgtable *iop = &data->iop; + int i = 0, num_entries, max_entries, unmap_idx_start; + + /* Something went horribly wrong and we ran out of page table */ + if (WARN_ON(lvl == REALM_MAX_LEVELS)) + return 0; + + unmap_idx_start = REALM_LVL_IDX(iova, lvl, data); + ptep += unmap_idx_start; + pte = READ_ONCE(*ptep); + if (WARN_ON(!pte)) + return 0; + + /* If the size matches this level, we're in the right place */ + if (size == REALM_BLOCK_SIZE(lvl, data)) { + max_entries = REALM_PTES_PER_TABLE(data) - unmap_idx_start; + num_entries = min_t(int, pgcount, max_entries); + + while (i < num_entries) { + pte = READ_ONCE(*ptep); + if (WARN_ON(!pte)) + break; + + *ptep = 0; + + if (!iopte_leaf(pte, lvl, iop->fmt)) { + /* Also flush any partial walks */ + io_pgtable_tlb_flush_walk(iop, iova + i * size, size, + REALM_GRANULE(data)); + __realm_ns_free_pgtable(data, lvl + 1, iopte_deref(pte, data)); + } else if (!iommu_iotlb_gather_queued(gather)) { + io_pgtable_tlb_add_page(iop, gather, iova + i * size, size); + } + + ptep++; + i++; + } + + return i * size; + } + + /* Keep on walkin' */ + ptep = iopte_deref(pte, data); + return realm_ns_unmap(data, gather, iova, size, pgcount, lvl + 1, ptep); +} + +static void realm_free_pgtable(struct io_pgtable *iop) +{ + struct realm_io_pgtable *data = io_pgtable_to_data(iop); + + if (!data->ns) { + __realm_free_pgtable(data, data->start_level, data->pgd, 0UL); + } else { + __realm_free_pgtable(data, data->start_level, data->ns_pgd, 0UL); + __realm_ns_free_pgtable(data, data->start_level, data->pgd); + } + + kfree(data); +} + +static phys_addr_t realm_ns_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct realm_io_pgtable *data = io_pgtable_ops_to_data(ops); + realm_iopte pte, *ptep = data->pgd; + int lvl = data->start_level; + + do { + /* Valid IOPTE pointer? */ + if (!ptep) + return 0; + + /* Grab the IOPTE we're interested in */ + ptep += REALM_LVL_IDX(iova, lvl, data); + pte = READ_ONCE(*ptep); + + /* Valid entry? */ + if (!pte) + return 0; + + /* Leaf entry? */ + if (iopte_leaf(pte, lvl, data->iop.fmt)) + goto found_translation; + + /* Take it to the next level */ + ptep = iopte_deref(pte, data); + } while (++lvl < REALM_MAX_LEVELS); + + /* Ran out of page tables to walk */ + return 0; + +found_translation: + iova &= (REALM_BLOCK_SIZE(lvl, data) - 1); + return iopte_to_paddr(pte, data) | iova; +} + +static int realm_map_pages(struct io_pgtable_ops *ops, unsigned long iova, + phys_addr_t paddr, size_t pgsize, size_t pgcount, + int iommu_prot, gfp_t gfp, size_t *mapped) +{ + struct realm_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + int ret, lvl = data->start_level; + realm_iopte prot; + long iaext = (s64)iova >> cfg->ias; + size_t r_mapped = 0; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize)) + return -EINVAL; + + if (WARN_ON(iaext || paddr >> cfg->oas)) + return -ERANGE; + + /* If no access, then nothing to do */ + if (!(iommu_prot & (IOMMU_READ | IOMMU_WRITE))) + return 0; + + prot = realm_prot_to_pte(data, iommu_prot); + + if (!data->ns) { + ret = realm_map(data, iova, paddr, pgsize, pgcount, prot, + lvl, (realm_iopte *)data->pgd, gfp, mapped); + } else { + ret = realm_map(data, iova, paddr, pgsize, pgcount, + prot | REALM_PTE_NS, lvl, + (realm_iopte *)data->ns_pgd, gfp, &r_mapped); + if (ret) + return ret; + + ret = realm_ns_map(data, iova, paddr, pgsize, pgcount, prot, + lvl, (realm_iopte *)data->pgd, gfp, mapped); + if (ret) + return ret; + /* + * Synchronise all PTE updates for the new mapping before there's + * a chance for anything to kick off a table walk for the new iova. + */ + wmb(); + + if (r_mapped != *mapped) { + pr_err("realm ns mapped entry not equal\n"); + ret = -EFAULT; + } + } + + return ret; +} + +static size_t realm_unmap_pages(struct io_pgtable_ops *ops, unsigned long iova, + size_t pgsize, size_t pgcount, + struct iommu_iotlb_gather *gather) +{ + struct realm_io_pgtable *data = io_pgtable_ops_to_data(ops); + struct io_pgtable_cfg *cfg = &data->iop.cfg; + long iaext = (s64)iova >> cfg->ias; + size_t realm_bytes, ret; + int sl = data->start_level; + + if (WARN_ON(!pgsize || (pgsize & cfg->pgsize_bitmap) != pgsize || !pgcount)) + return 0; + + if (WARN_ON(iaext)) + return 0; + + if (!data->ns) { + ret = realm_unmap(data, gather, iova, pgsize, pgcount, sl, + (realm_iopte *)data->pgd); + } else { + realm_bytes = realm_unmap(data, gather, iova, pgsize, pgcount, + sl, (realm_iopte *)data->ns_pgd); + ret = realm_ns_unmap(data, gather, iova, pgsize, pgcount, sl, + (realm_iopte *)data->pgd); + if (realm_bytes != ret) + ret = 0; + } + + return ret; +} + +static phys_addr_t realm_iova_to_phys(struct io_pgtable_ops *ops, + unsigned long iova) +{ + struct realm_io_pgtable *data = io_pgtable_ops_to_data(ops); + + if (!data->ns) + return __realm_iova_to_phys(ops, iova); + else + return realm_ns_iova_to_phys(ops, iova); +} + +static struct realm_io_pgtable *realm_alloc_pgtable(struct io_pgtable_cfg *cfg, + bool ns) +{ + struct realm_io_pgtable *data; + int levels, va_bits, pg_shift; + + cfg->pgsize_bitmap &= SZ_4K | SZ_2M | SZ_1G; + + if (!(cfg->pgsize_bitmap & SZ_4K)) + return NULL; + + if (cfg->ias > REALM_MAX_ADDR_BITS) + return NULL; + + if (cfg->oas > REALM_MAX_ADDR_BITS) + return NULL; + + data = kmalloc(sizeof(*data), GFP_KERNEL); + if (!data) + return NULL; + data->ns = ns; + + pg_shift = __ffs(cfg->pgsize_bitmap); + data->bits_per_level = pg_shift - ilog2(sizeof(realm_iopte)); + + va_bits = cfg->ias - pg_shift; + levels = DIV_ROUND_UP(va_bits, data->bits_per_level); + data->start_level = REALM_MAX_LEVELS - levels; + + /* Calculate the actual size of our pgd (without concatenation) */ + data->pgd_bits = va_bits - (data->bits_per_level * (levels - 1)); + + data->iop.ops = (struct io_pgtable_ops) { + .map_pages = realm_map_pages, + .unmap_pages = realm_unmap_pages, + .iova_to_phys = realm_iova_to_phys, + }; + + return data; +} + +static struct io_pgtable *__realm_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, + void *cookie, bool ns) +{ + u64 reg; + void *pgd; + struct realm_io_pgtable *data; + typeof(&cfg->arm_lpae_s1_cfg.tcr) tcr = &cfg->arm_lpae_s1_cfg.tcr; + + data = realm_alloc_pgtable(cfg, ns); + if (!data) + return NULL; + + /* TCR */ + tcr->sh = ARM_LPAE_TCR_SH_IS; + tcr->irgn = ARM_LPAE_TCR_RGN_WBWA; + tcr->orgn = ARM_LPAE_TCR_RGN_WBWA; + tcr->tg = ARM_LPAE_TCR_TG0_4K; + + switch (cfg->oas) { + case 32: + tcr->ips = ARM_LPAE_TCR_PS_32_BIT; + break; + case 36: + tcr->ips = ARM_LPAE_TCR_PS_36_BIT; + break; + case 40: + tcr->ips = ARM_LPAE_TCR_PS_40_BIT; + break; + case 42: + tcr->ips = ARM_LPAE_TCR_PS_42_BIT; + break; + case 44: + tcr->ips = ARM_LPAE_TCR_PS_44_BIT; + break; + case 48: + tcr->ips = ARM_LPAE_TCR_PS_48_BIT; + break; + case 52: + tcr->ips = ARM_LPAE_TCR_PS_52_BIT; + break; + default: + goto out_free_data; + } + + tcr->tsz = 64ULL - cfg->ias; + + /* MAIRs */ + reg = (REALM_MAIR_ATTR_NC + << REALM_MAIR_ATTR_SHIFT(REALM_MAIR_ATTR_IDX_NC)) | + (REALM_MAIR_ATTR_WBRWA + << REALM_MAIR_ATTR_SHIFT(REALM_MAIR_ATTR_IDX_CACHE)) | + (REALM_MAIR_ATTR_DEVICE + << REALM_MAIR_ATTR_SHIFT(REALM_MAIR_ATTR_IDX_DEV)) | + (REALM_MAIR_ATTR_INC_OWBRWA + << REALM_MAIR_ATTR_SHIFT(REALM_MAIR_ATTR_IDX_INC_OCACHE)); + + cfg->arm_lpae_s1_cfg.mair = reg; + pgd = __realm_alloc_pages(REALM_PGD_SIZE(data), GFP_KERNEL, cfg); + if (!pgd) + goto out_free_data; + + if (rmi_smmu_page_table_create(virt_to_phys(pgd), 0, + virt_to_phys(pgd), + REALM_PGD_SIZE(data), 0)) { + goto out_free_realm_page; + } + + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); + + if (!data->ns) { + data->pgd = pgd; + } else { + data->ns_pgd = pgd; + cfg->ns_ttbr = virt_to_phys(data->ns_pgd); + + data->pgd = realm_ns_alloc_pages(REALM_PGD_SIZE(data), + GFP_KERNEL, cfg, cookie); + if (!data->pgd) + goto out_free_realm_page; + } + + /* TTBR */ + cfg->arm_lpae_s1_cfg.ttbr = virt_to_phys(data->pgd); + return &data->iop; + +out_free_realm_page: + __realm_free_pages(pgd, REALM_PGD_SIZE(data), cfg); +out_free_data: + kfree(data); + return NULL; +} + +static struct io_pgtable *__realm_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, + void *cookie, bool ns) +{ + u64 sl; + void *pgd; + struct realm_io_pgtable *data; + typeof(&cfg->arm_lpae_s2_cfg.vtcr) vtcr = &cfg->arm_lpae_s2_cfg.vtcr; + + data = realm_alloc_pgtable(cfg, ns); + if (!data) + return NULL; + + /* + * Concatenate PGDs at level 1 if possible in order to reduce + * the depth of the stage-2 walk. + */ + if (data->start_level == 0) { + unsigned long pgd_pages; + + pgd_pages = REALM_PGD_SIZE(data) / sizeof(realm_iopte); + if (pgd_pages <= REALM_S2_MAX_CONCAT_PAGES) { + data->pgd_bits += data->bits_per_level; + data->start_level++; + } + } + + /* VTCR */ + vtcr->sh = ARM_LPAE_TCR_SH_IS; + vtcr->irgn = ARM_LPAE_TCR_RGN_WBWA; + vtcr->orgn = ARM_LPAE_TCR_RGN_WBWA; + vtcr->tg = ARM_LPAE_TCR_TG0_4K; + sl = data->start_level + 1; + + switch (cfg->oas) { + case 32: + vtcr->ps = ARM_LPAE_TCR_PS_32_BIT; + break; + case 36: + vtcr->ps = ARM_LPAE_TCR_PS_36_BIT; + break; + case 40: + vtcr->ps = ARM_LPAE_TCR_PS_40_BIT; + break; + case 42: + vtcr->ps = ARM_LPAE_TCR_PS_42_BIT; + break; + case 44: + vtcr->ps = ARM_LPAE_TCR_PS_44_BIT; + break; + case 48: + vtcr->ps = ARM_LPAE_TCR_PS_48_BIT; + break; + case 52: + vtcr->ps = ARM_LPAE_TCR_PS_52_BIT; + break; + default: + goto out_free_data; + } + + vtcr->tsz = 64ULL - cfg->ias; + vtcr->sl = ~sl & REALM_VTCR_SL0_MASK; + + pgd = __realm_alloc_pages(REALM_PGD_SIZE(data), GFP_KERNEL, cfg); + if (!pgd) + goto out_free_data; + + if (rmi_smmu_page_table_create(virt_to_phys(pgd), 0, + virt_to_phys(pgd), + REALM_PGD_SIZE(data), 0)) { + goto out_free_realm_page; + } + + /* Ensure the empty pgd is visible before any actual TTBR write */ + wmb(); + + if (!data->ns) { + data->pgd = pgd; + } else { + data->ns_pgd = pgd; + cfg->ns_ttbr = virt_to_phys(data->ns_pgd); + + data->pgd = realm_ns_alloc_pages(REALM_PGD_SIZE(data), + GFP_KERNEL, cfg, cookie); + if (!data->pgd) + goto out_free_realm_page; + } + + /* TTBR */ + cfg->arm_lpae_s2_cfg.vttbr = virt_to_phys(data->pgd); + return &data->iop; + +out_free_realm_page: + __realm_free_pages(pgd, REALM_PGD_SIZE(data), cfg); +out_free_data: + kfree(data); + return NULL; +} + +static struct io_pgtable * +realm_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) +{ + return __realm_alloc_pgtable_s1(cfg, cookie, false); +} + +static struct io_pgtable * +realm_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) +{ + return __realm_alloc_pgtable_s2(cfg, cookie, false); +} + +static struct io_pgtable * +realm_ns_alloc_pgtable_s1(struct io_pgtable_cfg *cfg, void *cookie) +{ + return __realm_alloc_pgtable_s1(cfg, cookie, true); +} + +static struct io_pgtable * +realm_ns_alloc_pgtable_s2(struct io_pgtable_cfg *cfg, void *cookie) +{ + return __realm_alloc_pgtable_s2(cfg, cookie, true); +} + +struct io_pgtable_init_fns io_pgtable_realm_s1_init_fns = { + .alloc = realm_alloc_pgtable_s1, + .free = realm_free_pgtable, +}; + +struct io_pgtable_init_fns io_pgtable_realm_s2_init_fns = { + .alloc = realm_alloc_pgtable_s2, + .free = realm_free_pgtable, +}; + +struct io_pgtable_init_fns io_pgtable_realm_ns_s1_init_fns = { + .alloc = realm_ns_alloc_pgtable_s1, + .free = realm_free_pgtable, +}; + +struct io_pgtable_init_fns io_pgtable_realm_ns_s2_init_fns = { + .alloc = realm_ns_alloc_pgtable_s2, + .free = realm_free_pgtable, +}; diff --git a/drivers/iommu/io-pgtable.c b/drivers/iommu/io-pgtable.c index 8841c1487f00..cd30ff0e94c7 100644 --- a/drivers/iommu/io-pgtable.c +++ b/drivers/iommu/io-pgtable.c @@ -32,6 +32,12 @@ io_pgtable_init_table[IO_PGTABLE_NUM_FMTS] = { [AMD_IOMMU_V1] = &io_pgtable_amd_iommu_v1_init_fns, [AMD_IOMMU_V2] = &io_pgtable_amd_iommu_v2_init_fns, #endif +#ifdef CONFIG_HISI_CCA_DA + [CCA_REALM_S1] = &io_pgtable_realm_s1_init_fns, + [CCA_REALM_S2] = &io_pgtable_realm_s2_init_fns, + [CCA_REALM_NS_S1] = &io_pgtable_realm_ns_s1_init_fns, + [CCA_REALM_NS_S2] = &io_pgtable_realm_ns_s2_init_fns, +#endif }; static int check_custom_allocator(enum io_pgtable_fmt fmt, diff --git a/drivers/iommu/iommu.c b/drivers/iommu/iommu.c index e9f9e8a23006..f85f15eca625 100644 --- a/drivers/iommu/iommu.c +++ b/drivers/iommu/iommu.c @@ -2770,6 +2770,18 @@ static int __init iommu_init(void) } core_initcall(iommu_init); +#ifdef CONFIG_HISI_CCA_DA +int iommu_enable_rme(struct iommu_domain *domain) +{ + if (domain->type != IOMMU_DOMAIN_UNMANAGED) + return -EINVAL; + if (!domain->ops->enable_rme) + return -EINVAL; + return domain->ops->enable_rme(domain); +} +EXPORT_SYMBOL_GPL(iommu_enable_rme); +#endif + int iommu_set_pgtable_quirks(struct iommu_domain *domain, unsigned long quirk) { diff --git a/drivers/iommu/iommufd/hw_pagetable.c b/drivers/iommu/iommufd/hw_pagetable.c index c92e575cf01e..34861589b263 100644 --- a/drivers/iommu/iommufd/hw_pagetable.c +++ b/drivers/iommu/iommufd/hw_pagetable.c @@ -138,6 +138,11 @@ iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, hwpt_paging->ioas = ioas; hwpt_paging->nest_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT; +#ifdef CONFIG_HISI_CCA_DA + if (ictx->realm) + flags |= IOMMU_HWPT_RME; +#endif + if (ops->domain_alloc_user) { hwpt->domain = ops->domain_alloc_user(idev->dev, flags, NULL, user_data); @@ -155,6 +160,14 @@ iommufd_hwpt_paging_alloc(struct iommufd_ctx *ictx, struct iommufd_ioas *ioas, } } +#ifdef CONFIG_HISI_CCA_DA + if (ictx->realm) { + rc = iommu_enable_rme(hwpt->domain); + if (rc) + goto out_abort; + } +#endif + /* * Set the coherency mode before we do iopt_table_add_domain() as some * iommus have a per-PTE bit that controls it and need to decide before diff --git a/drivers/iommu/iommufd/ioas.c b/drivers/iommu/iommufd/ioas.c index 2c4b2bb11e78..c567ea945789 100644 --- a/drivers/iommu/iommufd/ioas.c +++ b/drivers/iommu/iommufd/ioas.c @@ -380,6 +380,26 @@ static int iommufd_ioas_option_huge_pages(struct iommu_option *cmd, return -EOPNOTSUPP; } +int iommufd_option_realm(struct iommu_option *cmd, struct iommufd_ctx *ictx) +{ + if (cmd->op == IOMMU_OPTION_OP_GET) { + cmd->val64 = !ictx->realm; + return 0; + } + if (cmd->op == IOMMU_OPTION_OP_SET) { + if (cmd->val64 == 0) { + ictx->realm = 0; + return 0; + } + if (cmd->val64 == 1) { + ictx->realm = 1; + return 0; + } + return -EINVAL; + } + return -EOPNOTSUPP; +} + int iommufd_ioas_option(struct iommufd_ucmd *ucmd) { struct iommu_option *cmd = ucmd->cmd; diff --git a/drivers/iommu/iommufd/iommufd_private.h b/drivers/iommu/iommufd/iommufd_private.h index 85a2d4c1cd83..7491207ad3d6 100644 --- a/drivers/iommu/iommufd/iommufd_private.h +++ b/drivers/iommu/iommufd/iommufd_private.h @@ -28,6 +28,7 @@ struct iommufd_ctx { u8 account_mode; /* Compatibility with VFIO no iommu */ u8 no_iommu_mode; + u8 realm; struct iommufd_ioas *vfio_ioas; }; @@ -254,6 +255,7 @@ int iommufd_ioas_unmap(struct iommufd_ucmd *ucmd); int iommufd_ioas_option(struct iommufd_ucmd *ucmd); int iommufd_option_rlimit_mode(struct iommu_option *cmd, struct iommufd_ctx *ictx); +int iommufd_option_realm(struct iommu_option *cmd, struct iommufd_ctx *ictx); int iommufd_vfio_ioas(struct iommufd_ucmd *ucmd); int iommufd_check_iova_range(struct io_pagetable *iopt, diff --git a/drivers/iommu/iommufd/main.c b/drivers/iommu/iommufd/main.c index 3872abbd8729..24bae872a3dc 100644 --- a/drivers/iommu/iommufd/main.c +++ b/drivers/iommu/iommufd/main.c @@ -280,6 +280,9 @@ static int iommufd_option(struct iommufd_ucmd *ucmd) case IOMMU_OPTION_HUGE_PAGES: rc = iommufd_ioas_option(ucmd); break; + case IOMMU_OPTION_REALM: + rc = iommufd_option_realm(cmd, ucmd->ictx); + break; default: return -EOPNOTSUPP; } diff --git a/drivers/iommu/iommufd/vfio_compat.c b/drivers/iommu/iommufd/vfio_compat.c index 514aacd64009..707e861d63e7 100644 --- a/drivers/iommu/iommufd/vfio_compat.c +++ b/drivers/iommu/iommufd/vfio_compat.c @@ -283,6 +283,7 @@ static int iommufd_vfio_check_extension(struct iommufd_ctx *ictx, case VFIO_TYPE1_IOMMU: case VFIO_TYPE1v2_IOMMU: case VFIO_UNMAP_ALL: + case VFIO_TYPE1_IOMMU_RME: return 1; case VFIO_NOIOMMU_IOMMU: @@ -325,8 +326,8 @@ static int iommufd_vfio_set_iommu(struct iommufd_ctx *ictx, unsigned long type) return 0; } - if ((type != VFIO_TYPE1_IOMMU && type != VFIO_TYPE1v2_IOMMU) || - no_iommu_mode) + if ((type != VFIO_TYPE1_IOMMU && type != VFIO_TYPE1v2_IOMMU && + type != VFIO_TYPE1_IOMMU_RME) || no_iommu_mode) return -EINVAL; /* VFIO fails the set_iommu if there is no group */ @@ -344,6 +345,10 @@ static int iommufd_vfio_set_iommu(struct iommufd_ctx *ictx, unsigned long type) */ if (type == VFIO_TYPE1_IOMMU) rc = iopt_disable_large_pages(&ioas->iopt); + + if (type == VFIO_TYPE1_IOMMU_RME) + ictx->realm = 1; + iommufd_put_object(ictx, &ioas->obj); return rc; } diff --git a/drivers/irqchip/irq-gic-v3-its.c b/drivers/irqchip/irq-gic-v3-its.c index b6d07bf8c0ad..d0d6cc20c33c 100644 --- a/drivers/irqchip/irq-gic-v3-its.c +++ b/drivers/irqchip/irq-gic-v3-its.c @@ -40,6 +40,7 @@ #include #include +#include #ifdef CONFIG_ARCH_PHYTIUM #include @@ -438,6 +439,9 @@ static struct page *its_alloc_pages_node(int node, gfp_t gfp, if (virtcca_cvm_domain()) return virtcca_its_alloc_shared_pages_node(node, gfp, order); + if (is_realm_world()) + return realm_alloc_shared_pages(gfp, order); + page = alloc_pages_node(node, gfp, order); if (!page) @@ -468,6 +472,11 @@ static void its_free_pages(void *addr, unsigned int order) return; } + if (is_realm_world()) { + realm_free_shared_pages(addr, order); + return; + } + /* * If the memory cannot be encrypted again then we must leak the pages. * set_memory_encrypted() will already have WARNed. @@ -2400,7 +2409,7 @@ static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info) if (!is_v4(its_dev->its)) return -EINVAL; - guard(raw_spinlock)(&its_dev->event_map.vlpi_lock); + guard(raw_spinlock_irqsave)(&its_dev->event_map.vlpi_lock); /* Unmap request? */ if (!info) diff --git a/drivers/pci/msi/msi.c b/drivers/pci/msi/msi.c index 4eea161663b1..9cc3adca4d28 100644 --- a/drivers/pci/msi/msi.c +++ b/drivers/pci/msi/msi.c @@ -166,6 +166,10 @@ void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg) #ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && dev != NULL && is_cc_dev(pci_dev_id(dev))) return virtcca_pci_read_msi_msg(dev, msg, base); +#endif +#ifdef CONFIG_HISI_CCA_DA + if (rme_dev_pci_read_msi_msg(entry, msg)) + return; #endif msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR); msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR); @@ -232,6 +236,10 @@ static inline void pci_write_msg_msix(struct msi_desc *desc, struct msi_msg *msg #ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && virtcca_pci_write_msg_msi(desc, msg)) return; +#endif +#ifdef CONFIG_HISI_CCA_DA + if (rme_dev_pci_write_msg_msi(desc, msg)) + return; #endif writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR); writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR); @@ -620,7 +628,6 @@ static void __iomem *msix_map_region(struct pci_dev *dev, return NULL; } phys_addr = pci_resource_start(dev, bir) + table_offset; - return ioremap(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE); } @@ -658,6 +665,10 @@ void msix_prepare_msi_desc(struct pci_dev *dev, struct msi_desc *desc) #ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) return virtcca_msix_prepare_msi_desc(dev, desc, addr); +#endif +#ifdef CONFIG_HISI_CCA_DA + if (rme_dev_msix_prepare_msi_desc(dev, desc)) + return; #endif desc->pci.msix_ctrl = readl(addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } @@ -799,6 +810,10 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries, #ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && is_cc_dev(pci_dev_id(dev))) return virtcca_msix_mask_all_cc(dev, dev->msix_base, tsize, pci_dev_id(dev)); +#endif +#ifdef CONFIG_HISI_CCA_DA + if (rme_dev_msix_mask_all(dev, tsize)) + return 0; #endif msix_mask_all(dev->msix_base, tsize); pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0); diff --git a/drivers/pci/msi/msi.h b/drivers/pci/msi/msi.h index 20119a585690..f8442a128ce0 100644 --- a/drivers/pci/msi/msi.h +++ b/drivers/pci/msi/msi.h @@ -9,6 +9,12 @@ #endif #endif +#ifdef CONFIG_HISI_CCA_DA +#ifndef __GENKSYMS__ +#include +#endif +#endif + #define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1) int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type); @@ -46,7 +52,10 @@ static inline void pci_msix_write_vector_ctrl(struct msi_desc *desc, u32 ctrl) if (is_virtcca_cvm_enable() && virtcca_pci_msix_write_vector_ctrl(desc, ctrl)) return; #endif - +#ifdef CONFIG_HISI_CCA_DA + if (rme_dev_pci_msix_write_vector_ctrl(desc, ctrl)) + return; +#endif if (desc->pci.msi_attrib.can_mask) writel(ctrl, desc_addr + PCI_MSIX_ENTRY_VECTOR_CTRL); } @@ -59,6 +68,10 @@ static inline void pci_msix_mask(struct msi_desc *desc) #ifdef CONFIG_HISI_VIRTCCA_CODA if (is_virtcca_cvm_enable() && virtcca_pci_msix_mask(desc)) return; +#endif +#ifdef CONFIG_HISI_CCA_DA + if (rme_dev_pci_msix_mask(desc)) + return; #endif /* Flush write to device */ readl(desc->pci.mask_base); diff --git a/drivers/pci/pci.c b/drivers/pci/pci.c index 6743cce7532d..57ccebcde56c 100644 --- a/drivers/pci/pci.c +++ b/drivers/pci/pci.c @@ -35,6 +35,11 @@ #ifdef CONFIG_MACH_LOONGSON64 #include #endif +#ifdef CONFIG_HISI_CCA_DA +#ifndef __GENKSYMS__ +#include +#endif +#endif #include "pci.h" DEFINE_MUTEX(pci_slot_mutex); @@ -4091,6 +4096,10 @@ static int __pci_request_region(struct pci_dev *pdev, int bar, pci_resource_len(pdev, bar), res_name, exclusive)) goto err_out; +#ifdef CONFIG_HISI_CCA_DA + if (ccada_init_mem_region(pdev, bar)) + goto err_out; +#endif } dr = find_pci_dr(pdev); diff --git a/drivers/vfio/pci/vfio_pci.c b/drivers/vfio/pci/vfio_pci.c index e727941f589d..ab808906d2b8 100644 --- a/drivers/vfio/pci/vfio_pci.c +++ b/drivers/vfio/pci/vfio_pci.c @@ -24,6 +24,9 @@ #include #include #include +#ifdef CONFIG_HISI_CCA_DA +#include +#endif #include "vfio_pci_priv.h" @@ -100,6 +103,35 @@ static bool vfio_pci_is_denylisted(struct pci_dev *pdev) return true; } +#ifdef CONFIG_HISI_CCA_DA +static int rme_vfio_pci_open_device(struct vfio_pci_core_device *vdev) +{ + struct pci_dev *pdev = vdev->pdev; + int ret; + + ret = kvm_rme_dev_assign(pdev, vdev->vdev.kvm); + if (ret) + return ret; + + ret = kvm_rme_dev_attach(pdev, vdev->vdev.kvm); + if (ret) { + kvm_rme_dev_unassign(pdev, vdev->vdev.kvm); + return ret; + } + + ret = vfio_pci_core_enable(vdev); + if (ret) { + kvm_rme_dev_detach(pdev, vdev->vdev.kvm); + kvm_rme_dev_unassign(pdev, vdev->vdev.kvm); + return ret; + } + + vfio_pci_core_finish_enable(vdev); + + return 0; +} +#endif + static int vfio_pci_open_device(struct vfio_device *core_vdev) { struct vfio_pci_core_device *vdev = @@ -107,6 +139,12 @@ static int vfio_pci_open_device(struct vfio_device *core_vdev) struct pci_dev *pdev = vdev->pdev; int ret; +#ifdef CONFIG_HISI_CCA_DA + /* assign dev to vm in rme support platform */ + if (is_support_rme() && vdev->vdev.kvm) + return rme_vfio_pci_open_device(vdev); +#endif + ret = vfio_pci_core_enable(vdev); if (ret) return ret; diff --git a/drivers/vfio/pci/vfio_pci_core.c b/drivers/vfio/pci/vfio_pci_core.c index 9c130f83b21b..6757b9192811 100644 --- a/drivers/vfio/pci/vfio_pci_core.c +++ b/drivers/vfio/pci/vfio_pci_core.c @@ -34,6 +34,9 @@ #ifdef CONFIG_HISI_VIRTCCA_CODA #include #endif +#ifdef CONFIG_HISI_CCA_DA +#include +#endif #include "vfio_pci_priv.h" @@ -713,6 +716,13 @@ void vfio_pci_core_close_device(struct vfio_device *core_vdev) vdev->req_trigger = NULL; } mutex_unlock(&vdev->igate); + +#ifdef CONFIG_HISI_CCA_DA + if (is_support_rme() && vdev->vdev.kvm) { + kvm_rme_dev_detach(vdev->pdev, vdev->vdev.kvm); + kvm_rme_dev_unassign(vdev->pdev, vdev->vdev.kvm); + } +#endif } EXPORT_SYMBOL_GPL(vfio_pci_core_close_device); diff --git a/drivers/vfio/vfio_iommu_type1.c b/drivers/vfio/vfio_iommu_type1.c index 000d6a4bd06d..8fd3bc2ec213 100644 --- a/drivers/vfio/vfio_iommu_type1.c +++ b/drivers/vfio/vfio_iommu_type1.c @@ -79,6 +79,7 @@ struct vfio_iommu { uint64_t num_non_pinned_groups; uint64_t num_non_hwdbm_domains; bool v2; + bool rme; bool dirty_page_tracking; struct list_head emulated_iommu_groups; bool dirty_log_get_no_clear; @@ -2633,9 +2634,12 @@ static int vfio_iommu_type1_attach_group(void *iommu_data, if (!domain->domain) goto out_free_domain; -#ifdef CONFIG_HISI_VIRTCCA_CODA - if (is_virtcca_cvm_enable() && iommu->secure) - domain->domain->secure = true; +#ifdef CONFIG_HISI_CCA_DA + if (iommu->rme) { + ret = iommu_enable_rme(domain->domain); + if (ret) + goto out_domain; + } #endif ret = iommu_attach_group(domain->domain, group->iommu_group); @@ -3013,11 +3017,17 @@ static void *vfio_iommu_type1_open(unsigned long arg) iommu->secure = true; break; #endif + case VFIO_TYPE1_IOMMU_RME: + iommu->v2 = true; + iommu->rme = true; + break; default: kfree(iommu); return ERR_PTR(-EINVAL); } + + INIT_LIST_HEAD(&iommu->domain_list); INIT_LIST_HEAD(&iommu->iova_list); iommu->dma_list = RB_ROOT; @@ -3108,6 +3118,7 @@ static int vfio_iommu_type1_check_extension(struct vfio_iommu *iommu, case VFIO_TYPE1v2_S_IOMMU: #endif case VFIO_UNMAP_ALL: + case VFIO_TYPE1_IOMMU_RME: return 1; case VFIO_UPDATE_VADDR: /* diff --git a/drivers/virtio/virtio_mmio.c b/drivers/virtio/virtio_mmio.c index fb4abd2afc46..15293d9d2c7f 100644 --- a/drivers/virtio/virtio_mmio.c +++ b/drivers/virtio/virtio_mmio.c @@ -72,6 +72,7 @@ #include #include +#include /* The alignment to use between consumer and producer parts of vring. * Currently hardcoded to the page size. */ @@ -620,6 +621,8 @@ static int virtio_mmio_probe(struct platform_device *pdev) int rc; enable_swiotlb_for_cvm_dev(&pdev->dev, true); + enable_swiotlb_for_realm_dev(&pdev->dev, true); + vm_dev = kzalloc(sizeof(*vm_dev), GFP_KERNEL); if (!vm_dev) return -ENOMEM; diff --git a/drivers/virtio/virtio_pci_common.c b/drivers/virtio/virtio_pci_common.c index 95d9a3aed68c..3a796407a02c 100644 --- a/drivers/virtio/virtio_pci_common.c +++ b/drivers/virtio/virtio_pci_common.c @@ -16,6 +16,8 @@ #include "virtio_pci_common.h" +#include + static bool force_legacy = false; #if IS_ENABLED(CONFIG_VIRTIO_PCI_LEGACY) @@ -526,7 +528,7 @@ static int virtio_pci_probe(struct pci_dev *pci_dev, int rc; enable_swiotlb_for_cvm_dev(&pci_dev->dev, true); - + enable_swiotlb_for_realm_dev(&pci_dev->dev, true); /* allocate our structure and fill it out */ vp_dev = kzalloc(sizeof(struct virtio_pci_device), GFP_KERNEL); if (!vp_dev) diff --git a/include/linux/io-pgtable.h b/include/linux/io-pgtable.h index 24640a3151ea..1a77d0d1de7c 100644 --- a/include/linux/io-pgtable.h +++ b/include/linux/io-pgtable.h @@ -19,6 +19,10 @@ enum io_pgtable_fmt { AMD_IOMMU_V2, APPLE_DART, APPLE_DART2, + CCA_REALM_S1, + CCA_REALM_S2, + CCA_REALM_NS_S1, + CCA_REALM_NS_S2, IO_PGTABLE_NUM_FMTS, }; @@ -183,6 +187,11 @@ struct io_pgtable_cfg { u32 n_ttbrs; } apple_dart_cfg; }; + +#ifdef CONFIG_HISI_CCA_DA + /* For support pcipc multi vf */ + u64 ns_ttbr; +#endif }; /** @@ -322,5 +331,9 @@ extern struct io_pgtable_init_fns io_pgtable_arm_mali_lpae_init_fns; extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v1_init_fns; extern struct io_pgtable_init_fns io_pgtable_amd_iommu_v2_init_fns; extern struct io_pgtable_init_fns io_pgtable_apple_dart_init_fns; +extern struct io_pgtable_init_fns io_pgtable_realm_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_realm_s2_init_fns; +extern struct io_pgtable_init_fns io_pgtable_realm_ns_s1_init_fns; +extern struct io_pgtable_init_fns io_pgtable_realm_ns_s2_init_fns; #endif /* __IO_PGTABLE_H */ diff --git a/include/linux/iommu.h b/include/linux/iommu.h index 0dea4c637fd2..8b1e0adc72f8 100644 --- a/include/linux/iommu.h +++ b/include/linux/iommu.h @@ -827,7 +827,11 @@ struct iommu_domain_ops { KABI_USE(1, struct iommu_domain * (*get_msi_mapping_domain)(struct iommu_domain *domain)) +#ifdef CONFIG_HISI_CCA_DA + KABI_USE(2, int (*enable_rme)(struct iommu_domain *domain)) +#else KABI_RESERVE(2) +#endif KABI_RESERVE(3) KABI_RESERVE(4) KABI_RESERVE(5) @@ -1030,6 +1034,9 @@ extern int iommu_group_id(struct iommu_group *group); struct kset *iommu_get_group_kset(void); extern struct iommu_domain *iommu_group_default_domain(struct iommu_group *); +#ifdef CONFIG_HISI_CCA_DA +int iommu_enable_rme(struct iommu_domain *domain); +#endif int iommu_set_pgtable_quirks(struct iommu_domain *domain, unsigned long quirks); diff --git a/include/uapi/linux/iommufd.h b/include/uapi/linux/iommufd.h index 0be75c63dba8..163dbd93def1 100644 --- a/include/uapi/linux/iommufd.h +++ b/include/uapi/linux/iommufd.h @@ -268,7 +268,8 @@ struct iommu_ioas_unmap { /** * enum iommufd_option - ioctl(IOMMU_OPTION_RLIMIT_MODE) and - * ioctl(IOMMU_OPTION_HUGE_PAGES) + * ioctl(IOMMU_OPTION_HUGE_PAGES) and + * ioctl(IOMMU_OPTION_REALM) * @IOMMU_OPTION_RLIMIT_MODE: * Change how RLIMIT_MEMLOCK accounting works. The caller must have privilege * to invoke this. Value 0 (default) is user based accouting, 1 uses process @@ -278,10 +279,14 @@ struct iommu_ioas_unmap { * iommu mappings. Value 0 disables combining, everything is mapped to * PAGE_SIZE. This can be useful for benchmarking. This is a per-IOAS * option, the object_id must be the IOAS ID. + * @IOMMU_OPTION_REALM: + * Value 0 (default) means disable Realm Management Extension + * Value 1 means enable Realm Management Extension */ enum iommufd_option { IOMMU_OPTION_RLIMIT_MODE = 0, IOMMU_OPTION_HUGE_PAGES = 1, + IOMMU_OPTION_REALM = 2, }; /** @@ -361,11 +366,13 @@ struct iommu_vfio_ioas { * enforced on device attachment * @IOMMU_HWPT_FAULT_ID_VALID: The fault_id field of hwpt allocation data is * valid. + * @IOMMU_HWPT_RME: Realm management extension support */ enum iommufd_hwpt_alloc_flags { IOMMU_HWPT_ALLOC_NEST_PARENT = 1 << 0, IOMMU_HWPT_ALLOC_DIRTY_TRACKING = 1 << 1, IOMMU_HWPT_FAULT_ID_VALID = 1 << 2, + IOMMU_HWPT_RME = 1 << 3, }; /** diff --git a/include/uapi/linux/kvm.h b/include/uapi/linux/kvm.h index 51da8fe01061..3239bd862890 100644 --- a/include/uapi/linux/kvm.h +++ b/include/uapi/linux/kvm.h @@ -265,6 +265,9 @@ struct kvm_xen_exit { #define KVM_EXIT_RISCV_CSR 36 #define KVM_EXIT_NOTIFY 37 #define KVM_EXIT_LOONGARCH_IOCSR 38 +#ifndef __GENKSYMS__ +#define KVM_EXIT_ARM_RME_DEV 39 +#endif /* For KVM_EXIT_INTERNAL_ERROR */ /* Emulate instruction failed. */ @@ -518,6 +521,12 @@ struct kvm_run { #define KVM_NOTIFY_CONTEXT_INVALID (1 << 0) __u32 flags; } notify; +#ifndef __GENKSYMS__ + /* KVM_EXIT_ARM_RME_DEV */ + struct { + __u16 guest_dev_bdf; + } rme_dev; +#endif /* Fix the size of the union. */ char padding[256]; }; @@ -1653,6 +1662,8 @@ struct kvm_s390_ucas_mapping { /* Available with KVM_CAP_COUNTER_OFFSET */ #define KVM_ARM_SET_COUNTER_OFFSET _IOW(KVMIO, 0xb5, struct kvm_arm_counter_offset) +#define KVM_ARM_SMMU_READ_IPA _IOR(KVMIO, 0xb7, struct kvm_arm_read_ipa) + /* ioctl for SW vcpu init*/ #define KVM_SW64_VCPU_INIT _IO(KVMIO, 0xba) #define KVM_SW64_GET_VCB _IO(KVMIO, 0xbc) @@ -2478,4 +2489,14 @@ struct kvm_arm_rmm_psci_complete { #define KVM_ARM_VCPU_RMM_PSCI_COMPLETE _IOW(KVMIO, 0xd6, struct kvm_arm_rmm_psci_complete) +/* Available with KVM_CAP_ARM_RME, only for VMs with KVM_VM_TYPE_ARM_REALM */ +struct kvm_arm_rme_dev_validate { + __u16 dev_bdf; /* pci dev bdf in the host */ + __u16 guest_dev_bdf; /* pci dev bdf in the realm */ + bool vfio_dev; /* True if the host dev bdf refers to a VFIO dev */ + __u32 reserved[6]; +}; + +#define KVM_ARM_VCPU_RME_DEV_VALIDATE _IOW(KVMIO, 0xd3, struct kvm_arm_rme_dev_validate) + #endif /* __LINUX_KVM_H */ diff --git a/include/uapi/linux/vfio.h b/include/uapi/linux/vfio.h index a550d8470015..33ecfcc8a9b8 100644 --- a/include/uapi/linux/vfio.h +++ b/include/uapi/linux/vfio.h @@ -57,6 +57,9 @@ */ #define VFIO_UPDATE_VADDR 10 +/* Realm Management Extension */ +#define VFIO_TYPE1_IOMMU_RME 13 + /* * The vfio_iommu driver may support user clears dirty log manually, which means * dirty log can be requested to not cleared automatically after dirty log is diff --git a/kernel/dma/pool.c b/kernel/dma/pool.c index b481c48a31a6..fe71f8085000 100644 --- a/kernel/dma/pool.c +++ b/kernel/dma/pool.c @@ -13,6 +13,8 @@ #include #include +#include + static struct gen_pool *atomic_pool_dma __ro_after_init; static unsigned long pool_size_dma; static struct gen_pool *atomic_pool_dma32 __ro_after_init; @@ -87,6 +89,12 @@ static int atomic_pool_expand(struct gen_pool *pool, size_t pool_size, /* Cannot allocate larger than MAX_ORDER */ order = min(get_order(pool_size), MAX_ORDER); + if (is_realm_world()) { + page = realm_alloc_shared_pages(gfp, order); + if (!page) + goto out; + } + do { pool_size = 1 << (PAGE_SHIFT + order); if (cma_in_zone(gfp)) diff --git a/kernel/dma/swiotlb.c b/kernel/dma/swiotlb.c index b44ab7919428..5ab9d07c3de4 100644 --- a/kernel/dma/swiotlb.c +++ b/kernel/dma/swiotlb.c @@ -264,6 +264,9 @@ void __init swiotlb_update_mem_attributes(void) return; bytes = PAGE_ALIGN(mem->nslabs << IO_TLB_SHIFT); set_memory_decrypted((unsigned long)mem->vaddr, bytes >> PAGE_SHIFT); + + if (is_realm_world() && is_swiotlb_allocated()) + io_tlb_default_mem.for_alloc = true; } static void swiotlb_init_io_tlb_pool(struct io_tlb_pool *mem, phys_addr_t start, -- Gitee