diff options
Diffstat (limited to 'drivers/crypto/hisilicon')
| -rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_crypto.c | 185 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/hpre/hpre_main.c | 256 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/qm.c | 1843 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/qm.h | 17 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec.h | 23 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_crypto.c | 1036 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_crypto.h | 193 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/sec2/sec_main.c | 100 | ||||
| -rw-r--r-- | drivers/crypto/hisilicon/zip/zip_main.c | 99 |
9 files changed, 3158 insertions, 594 deletions
diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index a380087c83f7..a032c192ef1d 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -5,6 +5,7 @@ #include <crypto/dh.h> #include <crypto/ecc_curve.h> #include <crypto/ecdh.h> +#include <crypto/rng.h> #include <crypto/internal/akcipher.h> #include <crypto/internal/kpp.h> #include <crypto/internal/rsa.h> @@ -30,7 +31,6 @@ struct hpre_ctx; #define HPRE_DH_G_FLAG 0x02 #define HPRE_TRY_SEND_TIMES 100 #define HPRE_INVLD_REQ_ID (-1) -#define HPRE_DEV(ctx) (&((ctx)->qp->qm->pdev->dev)) #define HPRE_SQE_ALG_BITS 5 #define HPRE_SQE_DONE_SHIFT 30 @@ -39,12 +39,17 @@ struct hpre_ctx; #define HPRE_DFX_SEC_TO_US 1000000 #define HPRE_DFX_US_TO_NS 1000 +/* due to nist p521 */ +#define HPRE_ECC_MAX_KSZ 66 + /* size in bytes of the n prime */ #define HPRE_ECC_NIST_P192_N_SIZE 24 #define HPRE_ECC_NIST_P256_N_SIZE 32 +#define HPRE_ECC_NIST_P384_N_SIZE 48 /* size in bytes */ #define HPRE_ECC_HW256_KSZ_B 32 +#define HPRE_ECC_HW384_KSZ_B 48 typedef void (*hpre_cb)(struct hpre_ctx *ctx, void *sqe); @@ -102,6 +107,7 @@ struct hpre_curve25519_ctx { struct hpre_ctx { struct hisi_qp *qp; + struct device *dev; struct hpre_asym_request **req_list; struct hpre *hpre; spinlock_t req_lock; @@ -214,8 +220,7 @@ static int hpre_get_data_dma_addr(struct hpre_asym_request *hpre_req, struct scatterlist *data, unsigned int len, int is_src, dma_addr_t *tmp) { - struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = hpre_req->ctx->dev; enum dma_data_direction dma_dir; if (is_src) { @@ -239,7 +244,7 @@ static int hpre_prepare_dma_buf(struct hpre_asym_request *hpre_req, int is_src, dma_addr_t *tmp) { struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; void *ptr; int shift; @@ -293,11 +298,13 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, struct scatterlist *dst, struct scatterlist *src) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; struct hpre_sqe *sqe = &req->req; dma_addr_t tmp; tmp = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, tmp))) + return; if (src) { if (req->src) @@ -307,6 +314,8 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, } tmp = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, tmp))) + return; if (req->dst) { if (dst) @@ -321,16 +330,15 @@ static void hpre_hw_data_clr_all(struct hpre_ctx *ctx, static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, void **kreq) { - struct device *dev = HPRE_DEV(ctx); struct hpre_asym_request *req; unsigned int err, done, alg; int id; #define HPRE_NO_HW_ERR 0 #define HPRE_HW_TASK_DONE 3 -#define HREE_HW_ERR_MASK 0x7ff -#define HREE_SQE_DONE_MASK 0x3 -#define HREE_ALG_TYPE_MASK 0x1f +#define HREE_HW_ERR_MASK GENMASK(10, 0) +#define HREE_SQE_DONE_MASK GENMASK(1, 0) +#define HREE_ALG_TYPE_MASK GENMASK(4, 0) id = (int)le16_to_cpu(sqe->tag); req = ctx->req_list[id]; hpre_rm_req_from_ctx(req); @@ -346,7 +354,7 @@ static int hpre_alg_res_post_hf(struct hpre_ctx *ctx, struct hpre_sqe *sqe, return 0; alg = le32_to_cpu(sqe->dw0) & HREE_ALG_TYPE_MASK; - dev_err_ratelimited(dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", + dev_err_ratelimited(ctx->dev, "alg[0x%x] error: done[0x%x], etype[0x%x]\n", alg, done, err); return -EINVAL; @@ -361,6 +369,7 @@ static int hpre_ctx_set(struct hpre_ctx *ctx, struct hisi_qp *qp, int qlen) spin_lock_init(&ctx->req_lock); ctx->qp = qp; + ctx->dev = &qp->qm->pdev->dev; hpre = container_of(ctx->qp->qm, struct hpre, qm); ctx->hpre = hpre; @@ -524,6 +533,8 @@ static int hpre_msg_request_set(struct hpre_ctx *ctx, void *req, bool is_rsa) msg->key = cpu_to_le64(ctx->dh.dma_xa_p); } + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); msg->dw0 |= cpu_to_le32(0x1 << HPRE_SQE_DONE_SHIFT); msg->task_len1 = (ctx->key_sz >> HPRE_BITS_2_BYTES_SHIFT) - 1; h_req->ctx = ctx; @@ -618,14 +629,14 @@ static int hpre_is_dh_params_length_valid(unsigned int key_sz) case _HPRE_DH_GRP15: case _HPRE_DH_GRP16: return 0; + default: + return -EINVAL; } - - return -EINVAL; } static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz; if (params->p_size > HPRE_DH_MAX_P_SZ) @@ -664,7 +675,7 @@ static int hpre_dh_set_params(struct hpre_ctx *ctx, struct dh *params) static void hpre_dh_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; if (is_clear_all) @@ -877,18 +888,18 @@ static int hpre_rsa_set_n(struct hpre_ctx *ctx, const char *value, if (!hpre_rsa_key_size_is_support(ctx->key_sz)) return 0; - ctx->rsa.pubkey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1, + ctx->rsa.pubkey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_pubkey, GFP_KERNEL); if (!ctx->rsa.pubkey) return -ENOMEM; if (private) { - ctx->rsa.prikey = dma_alloc_coherent(HPRE_DEV(ctx), vlen << 1, + ctx->rsa.prikey = dma_alloc_coherent(ctx->dev, vlen << 1, &ctx->rsa.dma_prikey, GFP_KERNEL); if (!ctx->rsa.prikey) { - dma_free_coherent(HPRE_DEV(ctx), vlen << 1, + dma_free_coherent(ctx->dev, vlen << 1, ctx->rsa.pubkey, ctx->rsa.dma_pubkey); ctx->rsa.pubkey = NULL; @@ -950,7 +961,7 @@ static int hpre_crt_para_get(char *para, size_t para_sz, static int hpre_rsa_setkey_crt(struct hpre_ctx *ctx, struct rsa_key *rsa_key) { unsigned int hlf_ksz = ctx->key_sz >> 1; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; u64 offset; int ret; @@ -1008,7 +1019,7 @@ free_key: static void hpre_rsa_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all) { unsigned int half_key_sz = ctx->key_sz >> 1; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; if (is_clear_all) hisi_qm_stop_qp(ctx->qp); @@ -1179,7 +1190,7 @@ static void hpre_key_to_big_end(u8 *data, int len) static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, bool is_ecdh) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; @@ -1202,12 +1213,21 @@ static void hpre_ecc_clear_ctx(struct hpre_ctx *ctx, bool is_clear_all, hpre_ctx_clear(ctx, is_clear_all); } +/* + * The bits of 192/224/256/384/521 are supported by HPRE, + * and convert the bits like: + * bits<=256, bits=256; 256<bits<=384, bits=384; 384<bits<=576, bits=576; + * If the parameter bit width is insufficient, then we fill in the + * high-order zeros by soft, so TASK_LENGTH1 is 0x3/0x5/0x8; + */ static unsigned int hpre_ecdh_supported_curve(unsigned short id) { switch (id) { case ECC_CURVE_NIST_P192: case ECC_CURVE_NIST_P256: return HPRE_ECC_HW256_KSZ_B; + case ECC_CURVE_NIST_P384: + return HPRE_ECC_HW384_KSZ_B; default: break; } @@ -1272,6 +1292,8 @@ static unsigned int hpre_ecdh_get_curvesz(unsigned short id) return HPRE_ECC_NIST_P192_N_SIZE; case ECC_CURVE_NIST_P256: return HPRE_ECC_NIST_P256_N_SIZE; + case ECC_CURVE_NIST_P384: + return HPRE_ECC_NIST_P384_N_SIZE; default: break; } @@ -1281,7 +1303,7 @@ static unsigned int hpre_ecdh_get_curvesz(unsigned short id) static int hpre_ecdh_set_param(struct hpre_ctx *ctx, struct ecdh *params) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz, shift, curve_sz; int ret; @@ -1328,11 +1350,32 @@ static bool hpre_key_is_zero(char *key, unsigned short key_sz) return true; } +static int ecdh_gen_privkey(struct hpre_ctx *ctx, struct ecdh *params) +{ + struct device *dev = ctx->dev; + int ret; + + ret = crypto_get_default_rng(); + if (ret) { + dev_err(dev, "failed to get default rng, ret = %d!\n", ret); + return ret; + } + + ret = crypto_rng_get_bytes(crypto_default_rng, (u8 *)params->key, + params->key_size); + crypto_put_default_rng(); + if (ret) + dev_err(dev, "failed to get rng, ret = %d!\n", ret); + + return ret; +} + static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; + char key[HPRE_ECC_MAX_KSZ]; unsigned int sz, sz_shift; struct ecdh params; int ret; @@ -1342,6 +1385,15 @@ static int hpre_ecdh_set_secret(struct crypto_kpp *tfm, const void *buf, return -EINVAL; } + /* Use stdrng to generate private key */ + if (!params.key || !params.key_size) { + params.key = key; + params.key_size = hpre_ecdh_get_curvesz(ctx->curve_id); + ret = ecdh_gen_privkey(ctx, ¶ms); + if (ret) + return ret; + } + if (hpre_key_is_zero(params.key, params.key_size)) { dev_err(dev, "Invalid hpre key!\n"); return -EINVAL; @@ -1367,16 +1419,20 @@ static void hpre_ecdh_hw_data_clr_all(struct hpre_ctx *ctx, struct scatterlist *dst, struct scatterlist *src) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; struct hpre_sqe *sqe = &req->req; dma_addr_t dma; dma = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (src && req->src) dma_free_coherent(dev, ctx->key_sz << 2, req->src, dma); dma = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (req->dst) dma_free_coherent(dev, ctx->key_sz << 1, req->dst, dma); @@ -1431,6 +1487,8 @@ static int hpre_ecdh_msg_request_set(struct hpre_ctx *ctx, h_req->areq.ecdh = req; msg = &h_req->req; memset(msg, 0, sizeof(*msg)); + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); msg->key = cpu_to_le64(ctx->ecdh.dma_p); msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); @@ -1450,7 +1508,7 @@ static int hpre_ecdh_src_data_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int tmpshift; dma_addr_t dma = 0; void *ptr; @@ -1480,8 +1538,8 @@ static int hpre_ecdh_dst_data_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); - dma_addr_t dma = 0; + struct device *dev = ctx->dev; + dma_addr_t dma; if (unlikely(!data || !sg_is_last(data) || len != ctx->key_sz << 1)) { dev_err(dev, "data or data length is illegal!\n"); @@ -1503,7 +1561,7 @@ static int hpre_ecdh_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; void *tmp = kpp_request_ctx(req); struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); struct hpre_sqe *msg = &hpre_req->req; @@ -1568,6 +1626,15 @@ static int hpre_ecdh_nist_p256_init_tfm(struct crypto_kpp *tfm) return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); } +static int hpre_ecdh_nist_p384_init_tfm(struct crypto_kpp *tfm) +{ + struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); + + ctx->curve_id = ECC_CURVE_NIST_P384; + + return hpre_ctx_init(ctx, HPRE_V3_ECC_ALG_TYPE); +} + static void hpre_ecdh_exit_tfm(struct crypto_kpp *tfm) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); @@ -1609,7 +1676,7 @@ static void hpre_curve25519_fill_curve(struct hpre_ctx *ctx, const void *buf, static int hpre_curve25519_set_param(struct hpre_ctx *ctx, const void *buf, unsigned int len) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; unsigned int sz = ctx->key_sz; unsigned int shift = sz << 1; @@ -1634,7 +1701,7 @@ static int hpre_curve25519_set_secret(struct crypto_kpp *tfm, const void *buf, unsigned int len) { struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; int ret = -EINVAL; if (len != CURVE25519_KEY_SIZE || @@ -1662,16 +1729,20 @@ static void hpre_curve25519_hw_data_clr_all(struct hpre_ctx *ctx, struct scatterlist *dst, struct scatterlist *src) { - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; struct hpre_sqe *sqe = &req->req; dma_addr_t dma; dma = le64_to_cpu(sqe->in); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (src && req->src) dma_free_coherent(dev, ctx->key_sz, req->src, dma); dma = le64_to_cpu(sqe->out); + if (unlikely(dma_mapping_error(dev, dma))) + return; if (req->dst) dma_free_coherent(dev, ctx->key_sz, req->dst, dma); @@ -1722,6 +1793,8 @@ static int hpre_curve25519_msg_request_set(struct hpre_ctx *ctx, h_req->areq.curve25519 = req; msg = &h_req->req; memset(msg, 0, sizeof(*msg)); + msg->in = cpu_to_le64(DMA_MAPPING_ERROR); + msg->out = cpu_to_le64(DMA_MAPPING_ERROR); msg->key = cpu_to_le64(ctx->curve25519.dma_p); msg->dw0 |= cpu_to_le32(0x1U << HPRE_SQE_DONE_SHIFT); @@ -1752,7 +1825,7 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; u8 p[CURVE25519_KEY_SIZE] = { 0 }; const struct ecc_curve *curve; dma_addr_t dma = 0; @@ -1790,8 +1863,12 @@ static int hpre_curve25519_src_init(struct hpre_asym_request *hpre_req, * When src_data equals (2^255 - 19) ~ (2^255 - 1), it is out of p, * we get its modulus to p, and then use it. */ - if (memcmp(ptr, p, ctx->key_sz) >= 0) + if (memcmp(ptr, p, ctx->key_sz) == 0) { + dev_err(dev, "gx is p!\n"); + return -EINVAL; + } else if (memcmp(ptr, p, ctx->key_sz) > 0) { hpre_curve25519_src_modulo_p(ptr); + } hpre_req->src = ptr; msg->in = cpu_to_le64(dma); @@ -1807,8 +1884,8 @@ static int hpre_curve25519_dst_init(struct hpre_asym_request *hpre_req, { struct hpre_sqe *msg = &hpre_req->req; struct hpre_ctx *ctx = hpre_req->ctx; - struct device *dev = HPRE_DEV(ctx); - dma_addr_t dma = 0; + struct device *dev = ctx->dev; + dma_addr_t dma; if (!data || !sg_is_last(data) || len != ctx->key_sz) { dev_err(dev, "data or data length is illegal!\n"); @@ -1830,7 +1907,7 @@ static int hpre_curve25519_compute_value(struct kpp_request *req) { struct crypto_kpp *tfm = crypto_kpp_reqtfm(req); struct hpre_ctx *ctx = kpp_tfm_ctx(tfm); - struct device *dev = HPRE_DEV(ctx); + struct device *dev = ctx->dev; void *tmp = kpp_request_ctx(req); struct hpre_asym_request *hpre_req = PTR_ALIGN(tmp, HPRE_ALIGN_SZ); struct hpre_sqe *msg = &hpre_req->req; @@ -1940,7 +2017,7 @@ static struct kpp_alg ecdh_nist_p192 = { .cra_ctxsize = sizeof(struct hpre_ctx), .cra_priority = HPRE_CRYPTO_ALG_PRI, .cra_name = "ecdh-nist-p192", - .cra_driver_name = "hpre-ecdh", + .cra_driver_name = "hpre-ecdh-nist-p192", .cra_module = THIS_MODULE, }, }; @@ -1957,7 +2034,24 @@ static struct kpp_alg ecdh_nist_p256 = { .cra_ctxsize = sizeof(struct hpre_ctx), .cra_priority = HPRE_CRYPTO_ALG_PRI, .cra_name = "ecdh-nist-p256", - .cra_driver_name = "hpre-ecdh", + .cra_driver_name = "hpre-ecdh-nist-p256", + .cra_module = THIS_MODULE, + }, +}; + +static struct kpp_alg ecdh_nist_p384 = { + .set_secret = hpre_ecdh_set_secret, + .generate_public_key = hpre_ecdh_compute_value, + .compute_shared_secret = hpre_ecdh_compute_value, + .max_size = hpre_ecdh_max_size, + .init = hpre_ecdh_nist_p384_init_tfm, + .exit = hpre_ecdh_exit_tfm, + .reqsize = sizeof(struct hpre_asym_request) + HPRE_ALIGN_SZ, + .base = { + .cra_ctxsize = sizeof(struct hpre_ctx), + .cra_priority = HPRE_CRYPTO_ALG_PRI, + .cra_name = "ecdh-nist-p384", + .cra_driver_name = "hpre-ecdh-nist-p384", .cra_module = THIS_MODULE, }, }; @@ -1989,16 +2083,25 @@ static int hpre_register_ecdh(void) return ret; ret = crypto_register_kpp(&ecdh_nist_p256); - if (ret) { - crypto_unregister_kpp(&ecdh_nist_p192); - return ret; - } + if (ret) + goto unregister_ecdh_p192; + + ret = crypto_register_kpp(&ecdh_nist_p384); + if (ret) + goto unregister_ecdh_p256; return 0; + +unregister_ecdh_p256: + crypto_unregister_kpp(&ecdh_nist_p256); +unregister_ecdh_p192: + crypto_unregister_kpp(&ecdh_nist_p192); + return ret; } static void hpre_unregister_ecdh(void) { + crypto_unregister_kpp(&ecdh_nist_p384); crypto_unregister_kpp(&ecdh_nist_p256); crypto_unregister_kpp(&ecdh_nist_p192); } diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 046bc962c8b2..8b0640fb04be 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -36,7 +36,7 @@ #define HPRE_INT_MASK 0x301400 #define HPRE_INT_STATUS 0x301800 #define HPRE_CORE_INT_ENABLE 0 -#define HPRE_CORE_INT_DISABLE 0x003fffff +#define HPRE_CORE_INT_DISABLE GENMASK(21, 0) #define HPRE_RDCHN_INI_ST 0x301a00 #define HPRE_CLSTR_BASE 0x302000 #define HPRE_CORE_EN_OFFSET 0x04 @@ -50,6 +50,7 @@ #define HPRE_RAS_NFE_ENB 0x301414 #define HPRE_HAC_RAS_NFE_ENABLE 0x3ffffe #define HPRE_RAS_FE_ENB 0x301418 +#define HPRE_OOO_SHUTDOWN_SEL 0x301a3c #define HPRE_HAC_RAS_FE_ENABLE 0 #define HPRE_CORE_ENB (HPRE_CLSTR_BASE + HPRE_CORE_EN_OFFSET) @@ -57,7 +58,6 @@ #define HPRE_CORE_INI_STATUS (HPRE_CLSTR_BASE + HPRE_CORE_INI_STATUS_OFFSET) #define HPRE_HAC_ECC1_CNT 0x301a04 #define HPRE_HAC_ECC2_CNT 0x301a08 -#define HPRE_HAC_INT_STATUS 0x301800 #define HPRE_HAC_SOURCE_INT 0x301600 #define HPRE_CLSTR_ADDR_INTRVL 0x1000 #define HPRE_CLUSTER_INQURY 0x100 @@ -69,13 +69,17 @@ #define HPRE_DBGFS_VAL_MAX_LEN 20 #define HPRE_PCI_DEVICE_ID 0xa258 #define HPRE_PCI_VF_DEVICE_ID 0xa259 -#define HPRE_ADDR(qm, offset) ((qm)->io_base + (offset)) -#define HPRE_QM_USR_CFG_MASK 0xfffffffe -#define HPRE_QM_AXI_CFG_MASK 0xffff -#define HPRE_QM_VFG_AX_MASK 0xff -#define HPRE_BD_USR_MASK 0x3 -#define HPRE_CLUSTER_CORE_MASK_V2 0xf -#define HPRE_CLUSTER_CORE_MASK_V3 0xff +#define HPRE_QM_USR_CFG_MASK GENMASK(31, 1) +#define HPRE_QM_AXI_CFG_MASK GENMASK(15, 0) +#define HPRE_QM_VFG_AX_MASK GENMASK(7, 0) +#define HPRE_BD_USR_MASK GENMASK(1, 0) +#define HPRE_CLUSTER_CORE_MASK_V2 GENMASK(3, 0) +#define HPRE_CLUSTER_CORE_MASK_V3 GENMASK(7, 0) +#define HPRE_PREFETCH_CFG 0x301130 +#define HPRE_SVA_PREFTCH_DFX 0x30115C +#define HPRE_PREFETCH_ENABLE (~(BIT(0) | BIT(30))) +#define HPRE_PREFETCH_DISABLE BIT(30) +#define HPRE_SVA_DISABLE_READY (BIT(4) | BIT(8)) #define HPRE_AM_OOO_SHUTDOWN_ENB 0x301044 #define HPRE_AM_OOO_SHUTDOWN_ENABLE BIT(0) @@ -88,11 +92,7 @@ #define HPRE_QM_PM_FLR BIT(11) #define HPRE_QM_SRIOV_FLR BIT(12) -#define HPRE_CLUSTERS_NUM(qm) \ - (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : HPRE_CLUSTERS_NUM_V2) -#define HPRE_CLUSTER_CORE_MASK(qm) \ - (((qm)->ver >= QM_HW_V3) ? HPRE_CLUSTER_CORE_MASK_V3 :\ - HPRE_CLUSTER_CORE_MASK_V2) +#define HPRE_SHAPER_TYPE_RATE 128 #define HPRE_VIA_MSI_DSM 1 #define HPRE_SQE_MASK_OFFSET 8 #define HPRE_SQE_MASK_LEN 24 @@ -123,21 +123,49 @@ static const char * const hpre_debug_file_name[] = { }; static const struct hpre_hw_error hpre_hw_errors[] = { - { .int_msk = BIT(0), .msg = "core_ecc_1bit_err_int_set" }, - { .int_msk = BIT(1), .msg = "core_ecc_2bit_err_int_set" }, - { .int_msk = BIT(2), .msg = "dat_wb_poison_int_set" }, - { .int_msk = BIT(3), .msg = "dat_rd_poison_int_set" }, - { .int_msk = BIT(4), .msg = "bd_rd_poison_int_set" }, - { .int_msk = BIT(5), .msg = "ooo_ecc_2bit_err_int_set" }, - { .int_msk = BIT(6), .msg = "cluster1_shb_timeout_int_set" }, - { .int_msk = BIT(7), .msg = "cluster2_shb_timeout_int_set" }, - { .int_msk = BIT(8), .msg = "cluster3_shb_timeout_int_set" }, - { .int_msk = BIT(9), .msg = "cluster4_shb_timeout_int_set" }, - { .int_msk = GENMASK(15, 10), .msg = "ooo_rdrsp_err_int_set" }, - { .int_msk = GENMASK(21, 16), .msg = "ooo_wrrsp_err_int_set" }, - { .int_msk = BIT(22), .msg = "pt_rng_timeout_int_set"}, - { .int_msk = BIT(23), .msg = "sva_fsm_timeout_int_set"}, { + .int_msk = BIT(0), + .msg = "core_ecc_1bit_err_int_set" + }, { + .int_msk = BIT(1), + .msg = "core_ecc_2bit_err_int_set" + }, { + .int_msk = BIT(2), + .msg = "dat_wb_poison_int_set" + }, { + .int_msk = BIT(3), + .msg = "dat_rd_poison_int_set" + }, { + .int_msk = BIT(4), + .msg = "bd_rd_poison_int_set" + }, { + .int_msk = BIT(5), + .msg = "ooo_ecc_2bit_err_int_set" + }, { + .int_msk = BIT(6), + .msg = "cluster1_shb_timeout_int_set" + }, { + .int_msk = BIT(7), + .msg = "cluster2_shb_timeout_int_set" + }, { + .int_msk = BIT(8), + .msg = "cluster3_shb_timeout_int_set" + }, { + .int_msk = BIT(9), + .msg = "cluster4_shb_timeout_int_set" + }, { + .int_msk = GENMASK(15, 10), + .msg = "ooo_rdrsp_err_int_set" + }, { + .int_msk = GENMASK(21, 16), + .msg = "ooo_wrrsp_err_int_set" + }, { + .int_msk = BIT(22), + .msg = "pt_rng_timeout_int_set" + }, { + .int_msk = BIT(23), + .msg = "sva_fsm_timeout_int_set" + }, { /* sentinel */ } }; @@ -224,6 +252,18 @@ static u32 vfs_num; module_param_cb(vfs_num, &vfs_num_ops, &vfs_num, 0444); MODULE_PARM_DESC(vfs_num, "Number of VFs to enable(1-63), 0(default)"); +static inline int hpre_cluster_num(struct hisi_qm *qm) +{ + return (qm->ver >= QM_HW_V3) ? HPRE_CLUSTERS_NUM_V3 : + HPRE_CLUSTERS_NUM_V2; +} + +static inline int hpre_cluster_core_mask(struct hisi_qm *qm) +{ + return (qm->ver >= QM_HW_V3) ? + HPRE_CLUSTER_CORE_MASK_V3 : HPRE_CLUSTER_CORE_MASK_V2; +} + struct hisi_qp *hpre_create_qp(u8 type) { int node = cpu_to_node(smp_processor_id()); @@ -290,8 +330,8 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm) static int hpre_set_cluster(struct hisi_qm *qm) { - u32 cluster_core_mask = HPRE_CLUSTER_CORE_MASK(qm); - u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + u32 cluster_core_mask = hpre_cluster_core_mask(qm); + u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; unsigned long offset; u32 val = 0; @@ -302,10 +342,10 @@ static int hpre_set_cluster(struct hisi_qm *qm) /* clusters initiating */ writel(cluster_core_mask, - HPRE_ADDR(qm, offset + HPRE_CORE_ENB)); - writel(0x1, HPRE_ADDR(qm, offset + HPRE_CORE_INI_CFG)); - ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, offset + - HPRE_CORE_INI_STATUS), val, + qm->io_base + offset + HPRE_CORE_ENB); + writel(0x1, qm->io_base + offset + HPRE_CORE_INI_CFG); + ret = readl_relaxed_poll_timeout(qm->io_base + offset + + HPRE_CORE_INI_STATUS, val, ((val & cluster_core_mask) == cluster_core_mask), HPRE_REG_RD_INTVRL_US, @@ -329,11 +369,52 @@ static void disable_flr_of_bme(struct hisi_qm *qm) { u32 val; - val = readl(HPRE_ADDR(qm, QM_PEH_AXUSER_CFG)); + val = readl(qm->io_base + QM_PEH_AXUSER_CFG); val &= ~(HPRE_QM_BME_FLR | HPRE_QM_SRIOV_FLR); val |= HPRE_QM_PM_FLR; - writel(val, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG)); - writel(PEH_AXUSER_CFG_ENABLE, HPRE_ADDR(qm, QM_PEH_AXUSER_CFG_ENABLE)); + writel(val, qm->io_base + QM_PEH_AXUSER_CFG); + writel(PEH_AXUSER_CFG_ENABLE, qm->io_base + QM_PEH_AXUSER_CFG_ENABLE); +} + +static void hpre_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); + val &= HPRE_PREFETCH_ENABLE; + writel(val, qm->io_base + HPRE_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_PREFETCH_CFG, + val, !(val & HPRE_PREFETCH_DISABLE), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to open sva prefetch\n"); +} + +static void hpre_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + HPRE_PREFETCH_CFG); + val |= HPRE_PREFETCH_DISABLE; + writel(val, qm->io_base + HPRE_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_SVA_PREFTCH_DFX, + val, !(val & HPRE_SVA_DISABLE_READY), + HPRE_REG_RD_INTVRL_US, + HPRE_REG_RD_TMOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); } static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) @@ -342,33 +423,33 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) u32 val; int ret; - writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_ARUSER_M_CFG_ENABLE)); - writel(HPRE_QM_USR_CFG_MASK, HPRE_ADDR(qm, QM_AWUSER_M_CFG_ENABLE)); - writel_relaxed(HPRE_QM_AXI_CFG_MASK, HPRE_ADDR(qm, QM_AXI_M_CFG)); + writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_ARUSER_M_CFG_ENABLE); + writel(HPRE_QM_USR_CFG_MASK, qm->io_base + QM_AWUSER_M_CFG_ENABLE); + writel_relaxed(HPRE_QM_AXI_CFG_MASK, qm->io_base + QM_AXI_M_CFG); /* HPRE need more time, we close this interrupt */ - val = readl_relaxed(HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); + val = readl_relaxed(qm->io_base + HPRE_QM_ABNML_INT_MASK); val |= BIT(HPRE_TIMEOUT_ABNML_BIT); - writel_relaxed(val, HPRE_ADDR(qm, HPRE_QM_ABNML_INT_MASK)); + writel_relaxed(val, qm->io_base + HPRE_QM_ABNML_INT_MASK); if (qm->ver >= QM_HW_V3) writel(HPRE_RSA_ENB | HPRE_ECC_ENB, - HPRE_ADDR(qm, HPRE_TYPES_ENB)); + qm->io_base + HPRE_TYPES_ENB); else - writel(HPRE_RSA_ENB, HPRE_ADDR(qm, HPRE_TYPES_ENB)); - - writel(HPRE_QM_VFG_AX_MASK, HPRE_ADDR(qm, HPRE_VFG_AXCACHE)); - writel(0x0, HPRE_ADDR(qm, HPRE_BD_ENDIAN)); - writel(0x0, HPRE_ADDR(qm, HPRE_INT_MASK)); - writel(0x0, HPRE_ADDR(qm, HPRE_POISON_BYPASS)); - writel(0x0, HPRE_ADDR(qm, HPRE_COMM_CNT_CLR_CE)); - writel(0x0, HPRE_ADDR(qm, HPRE_ECC_BYPASS)); - - writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_ARUSR_CFG)); - writel(HPRE_BD_USR_MASK, HPRE_ADDR(qm, HPRE_BD_AWUSR_CFG)); - writel(0x1, HPRE_ADDR(qm, HPRE_RDCHN_INI_CFG)); - ret = readl_relaxed_poll_timeout(HPRE_ADDR(qm, HPRE_RDCHN_INI_ST), val, - val & BIT(0), + writel(HPRE_RSA_ENB, qm->io_base + HPRE_TYPES_ENB); + + writel(HPRE_QM_VFG_AX_MASK, qm->io_base + HPRE_VFG_AXCACHE); + writel(0x0, qm->io_base + HPRE_BD_ENDIAN); + writel(0x0, qm->io_base + HPRE_INT_MASK); + writel(0x0, qm->io_base + HPRE_POISON_BYPASS); + writel(0x0, qm->io_base + HPRE_COMM_CNT_CLR_CE); + writel(0x0, qm->io_base + HPRE_ECC_BYPASS); + + writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_ARUSR_CFG); + writel(HPRE_BD_USR_MASK, qm->io_base + HPRE_BD_AWUSR_CFG); + writel(0x1, qm->io_base + HPRE_RDCHN_INI_CFG); + ret = readl_relaxed_poll_timeout(qm->io_base + HPRE_RDCHN_INI_ST, val, + val & BIT(0), HPRE_REG_RD_INTVRL_US, HPRE_REG_RD_TMOUT_US); if (ret) { @@ -397,7 +478,7 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) static void hpre_cnt_regs_clear(struct hisi_qm *qm) { - u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + u8 clusters_num = hpre_cluster_num(qm); unsigned long offset; int i; @@ -413,36 +494,49 @@ static void hpre_cnt_regs_clear(struct hisi_qm *qm) hisi_qm_debug_regs_clear(qm); } -static void hpre_hw_error_disable(struct hisi_qm *qm) +static void hpre_master_ooo_ctrl(struct hisi_qm *qm, bool enable) { - u32 val; + u32 val1, val2; + + val1 = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + if (enable) { + val1 |= HPRE_AM_OOO_SHUTDOWN_ENABLE; + val2 = HPRE_HAC_RAS_NFE_ENABLE; + } else { + val1 &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + HPRE_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); +} +static void hpre_hw_error_disable(struct hisi_qm *qm) +{ /* disable hpre hw error interrupts */ writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_INT_MASK); - /* disable HPRE block master OOO when m-bit error occur */ - val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); - val &= ~HPRE_AM_OOO_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + /* disable HPRE block master OOO when nfe occurs on Kunpeng930 */ + hpre_master_ooo_ctrl(qm, false); } static void hpre_hw_error_enable(struct hisi_qm *qm) { - u32 val; - /* clear HPRE hw error source if having */ writel(HPRE_CORE_INT_DISABLE, qm->io_base + HPRE_HAC_SOURCE_INT); - /* enable hpre hw error interrupts */ - writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK); + /* configure error type */ writel(HPRE_HAC_RAS_CE_ENABLE, qm->io_base + HPRE_RAS_CE_ENB); writel(HPRE_HAC_RAS_NFE_ENABLE, qm->io_base + HPRE_RAS_NFE_ENB); writel(HPRE_HAC_RAS_FE_ENABLE, qm->io_base + HPRE_RAS_FE_ENB); - /* enable HPRE block master OOO when m-bit error occur */ - val = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); - val |= HPRE_AM_OOO_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); + /* enable HPRE block master OOO when nfe occurs on Kunpeng930 */ + hpre_master_ooo_ctrl(qm, true); + + /* enable hpre hw error interrupts */ + writel(HPRE_CORE_INT_ENABLE, qm->io_base + HPRE_INT_MASK); } static inline struct hisi_qm *hpre_file_to_qm(struct hpre_debugfs_file *file) @@ -650,7 +744,7 @@ static int hpre_pf_comm_regs_debugfs_init(struct hisi_qm *qm) static int hpre_cluster_debugfs_init(struct hisi_qm *qm) { - u8 clusters_num = HPRE_CLUSTERS_NUM(qm); + u8 clusters_num = hpre_cluster_num(qm); struct device *dev = &qm->pdev->dev; char buf[HPRE_DBGFS_VAL_MAX_LEN]; struct debugfs_regset32 *regset; @@ -788,7 +882,7 @@ static void hpre_log_hw_error(struct hisi_qm *qm, u32 err_sts) static u32 hpre_get_hw_err_status(struct hisi_qm *qm) { - return readl(qm->io_base + HPRE_HAC_INT_STATUS); + return readl(qm->io_base + HPRE_INT_STATUS); } static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) @@ -802,9 +896,9 @@ static void hpre_open_axi_master_ooo(struct hisi_qm *qm) value = readl(qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); writel(value & ~HPRE_AM_OOO_SHUTDOWN_ENABLE, - HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB)); + qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); writel(value | HPRE_AM_OOO_SHUTDOWN_ENABLE, - HPRE_ADDR(qm, HPRE_AM_OOO_SHUTDOWN_ENB)); + qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); } static void hpre_err_info_init(struct hisi_qm *qm) @@ -829,6 +923,8 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .clear_dev_hw_err_status = hpre_clear_hw_err_status, .log_dev_hw_err = hpre_log_hw_error, .open_axi_master_ooo = hpre_open_axi_master_ooo, + .open_sva_prefetch = hpre_open_sva_prefetch, + .close_sva_prefetch = hpre_close_sva_prefetch, .err_info_init = hpre_err_info_init, }; @@ -841,6 +937,8 @@ static int hpre_pf_probe_init(struct hpre *hpre) if (ret) return ret; + hpre_open_sva_prefetch(qm); + qm->err_ini = &hpre_err_ini; qm->err_ini->err_info_init(qm); hisi_qm_dev_err_init(qm); @@ -850,6 +948,7 @@ static int hpre_pf_probe_init(struct hpre *hpre) static int hpre_probe_init(struct hpre *hpre) { + u32 type_rate = HPRE_SHAPER_TYPE_RATE; struct hisi_qm *qm = &hpre->qm; int ret; @@ -857,6 +956,11 @@ static int hpre_probe_init(struct hpre *hpre) ret = hpre_pf_probe_init(hpre); if (ret) return ret; + /* Enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + qm->type_rate = type_rate; + } } return 0; diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index ce439a0c66c9..1d67f94a1d56 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -25,9 +25,11 @@ #define QM_IRQ_NUM_V1 1 #define QM_IRQ_NUM_PF_V2 4 #define QM_IRQ_NUM_VF_V2 2 +#define QM_IRQ_NUM_VF_V3 3 #define QM_EQ_EVENT_IRQ_VECTOR 0 #define QM_AEQ_EVENT_IRQ_VECTOR 1 +#define QM_CMD_EVENT_IRQ_VECTOR 2 #define QM_ABNORMAL_EVENT_IRQ_VECTOR 3 /* mailbox */ @@ -39,6 +41,8 @@ #define QM_MB_CMD_CQC_BT 0x5 #define QM_MB_CMD_SQC_VFT_V2 0x6 #define QM_MB_CMD_STOP_QP 0x8 +#define QM_MB_CMD_SRC 0xc +#define QM_MB_CMD_DST 0xd #define QM_MB_CMD_SEND_BASE 0x300 #define QM_MB_EVENT_SHIFT 8 @@ -46,6 +50,9 @@ #define QM_MB_OP_SHIFT 14 #define QM_MB_CMD_DATA_ADDR_L 0x304 #define QM_MB_CMD_DATA_ADDR_H 0x308 +#define QM_MB_PING_ALL_VFS 0xffff +#define QM_MB_CMD_DATA_SHIFT 32 +#define QM_MB_CMD_DATA_MASK GENMASK(31, 0) /* sqc shift */ #define QM_SQ_HOP_NUM_SHIFT 0 @@ -95,6 +102,7 @@ #define QM_DOORBELL_SQ_CQ_BASE_V2 0x1000 #define QM_DOORBELL_EQ_AEQ_BASE_V2 0x2000 #define QM_QUE_ISO_CFG_V 0x0030 +#define QM_PAGE_SIZE 0x0034 #define QM_QUE_ISO_EN 0x100154 #define QM_CAPBILITY 0x100158 #define QM_QP_NUN_MASK GENMASK(10, 0) @@ -155,11 +163,15 @@ #define QM_RAS_CE_THRESHOLD 0x1000f8 #define QM_RAS_CE_TIMES_PER_IRQ 1 #define QM_RAS_MSI_INT_SEL 0x1040f4 +#define QM_OOO_SHUTDOWN_SEL 0x1040f8 #define QM_RESET_WAIT_TIMEOUT 400 #define QM_PEH_VENDOR_ID 0x1000d8 #define ACC_VENDOR_ID_VALUE 0x5a5a #define QM_PEH_DFX_INFO0 0x1000fc +#define QM_PEH_DFX_INFO1 0x100100 +#define QM_PEH_DFX_MASK (BIT(0) | BIT(2)) +#define QM_PEH_MSI_FINISH_MASK GENMASK(19, 16) #define ACC_PEH_SRIOV_CTRL_VF_MSE_SHIFT 3 #define ACC_PEH_MSI_DISABLE GENMASK(31, 0) #define ACC_MASTER_GLOBAL_CTRL_SHUTDOWN 0x1 @@ -170,6 +182,31 @@ #define QM_RAS_NFE_MBIT_DISABLE ~QM_ECC_MBIT #define ACC_AM_ROB_ECC_INT_STS 0x300104 #define ACC_ROB_ECC_ERR_MULTPL BIT(1) +#define QM_MSI_CAP_ENABLE BIT(16) + +/* interfunction communication */ +#define QM_IFC_READY_STATUS 0x100128 +#define QM_IFC_C_STS_M 0x10012C +#define QM_IFC_INT_SET_P 0x100130 +#define QM_IFC_INT_CFG 0x100134 +#define QM_IFC_INT_SOURCE_P 0x100138 +#define QM_IFC_INT_SOURCE_V 0x0020 +#define QM_IFC_INT_MASK 0x0024 +#define QM_IFC_INT_STATUS 0x0028 +#define QM_IFC_INT_SET_V 0x002C +#define QM_IFC_SEND_ALL_VFS GENMASK(6, 0) +#define QM_IFC_INT_SOURCE_CLR GENMASK(63, 0) +#define QM_IFC_INT_SOURCE_MASK BIT(0) +#define QM_IFC_INT_DISABLE BIT(0) +#define QM_IFC_INT_STATUS_MASK BIT(0) +#define QM_IFC_INT_SET_MASK BIT(0) +#define QM_WAIT_DST_ACK 10 +#define QM_MAX_PF_WAIT_COUNT 10 +#define QM_MAX_VF_WAIT_COUNT 40 +#define QM_VF_RESET_WAIT_US 20000 +#define QM_VF_RESET_WAIT_CNT 3000 +#define QM_VF_RESET_WAIT_TIMEOUT_US \ + (QM_VF_RESET_WAIT_US * QM_VF_RESET_WAIT_CNT) #define QM_DFX_MB_CNT_VF 0x104010 #define QM_DFX_DB_CNT_VF 0x104020 @@ -205,6 +242,33 @@ #define QM_DRIVER_REMOVING 0 #define QM_RST_SCHED 1 #define QM_RESETTING 2 +#define QM_QOS_PARAM_NUM 2 +#define QM_QOS_VAL_NUM 1 +#define QM_QOS_BDF_PARAM_NUM 4 +#define QM_QOS_MAX_VAL 1000 +#define QM_QOS_RATE 100 +#define QM_QOS_EXPAND_RATE 1000 +#define QM_SHAPER_CIR_B_MASK GENMASK(7, 0) +#define QM_SHAPER_CIR_U_MASK GENMASK(10, 8) +#define QM_SHAPER_CIR_S_MASK GENMASK(14, 11) +#define QM_SHAPER_FACTOR_CIR_U_SHIFT 8 +#define QM_SHAPER_FACTOR_CIR_S_SHIFT 11 +#define QM_SHAPER_FACTOR_CBS_B_SHIFT 15 +#define QM_SHAPER_FACTOR_CBS_S_SHIFT 19 +#define QM_SHAPER_CBS_B 1 +#define QM_SHAPER_CBS_S 16 +#define QM_SHAPER_VFT_OFFSET 6 +#define WAIT_FOR_QOS_VF 100 +#define QM_QOS_MIN_ERROR_RATE 5 +#define QM_QOS_TYPICAL_NUM 8 +#define QM_SHAPER_MIN_CBS_S 8 +#define QM_QOS_TICK 0x300U +#define QM_QOS_DIVISOR_CLK 0x1f40U +#define QM_QOS_MAX_CIR_B 200 +#define QM_QOS_MIN_CIR_B 100 +#define QM_QOS_MAX_CIR_U 6 +#define QM_QOS_MAX_CIR_S 11 +#define QM_QOS_VAL_MAX_LEN 32 #define QM_MK_CQC_DW3_V1(hop_num, pg_sz, buf_sz, cqe_sz) \ (((hop_num) << QM_CQ_HOP_NUM_SHIFT) | \ @@ -245,6 +309,7 @@ enum vft_type { SQC_VFT = 0, CQC_VFT, + SHAPER_VFT, }; enum acc_err_result { @@ -253,6 +318,23 @@ enum acc_err_result { ACC_ERR_RECOVERED, }; +enum qm_alg_type { + ALG_TYPE_0, + ALG_TYPE_1, +}; + +enum qm_mb_cmd { + QM_PF_FLR_PREPARE = 0x01, + QM_PF_SRST_PREPARE, + QM_PF_RESET_DONE, + QM_VF_PREPARE_DONE, + QM_VF_PREPARE_FAIL, + QM_VF_START_DONE, + QM_VF_START_FAIL, + QM_PF_SET_QOS, + QM_VF_GET_QOS, +}; + struct qm_cqe { __le32 rsvd0; __le16 cmd_id; @@ -351,6 +433,9 @@ struct hisi_qm_hw_ops { void (*hw_error_uninit)(struct hisi_qm *qm); enum acc_err_result (*hw_error_handle)(struct hisi_qm *qm); int (*stop_qp)(struct hisi_qp *qp); + int (*set_msi)(struct hisi_qm *qm, bool set); + int (*ping_all_vfs)(struct hisi_qm *qm, u64 cmd); + int (*ping_pf)(struct hisi_qm *qm, u64 cmd); }; struct qm_dfx_item { @@ -412,6 +497,11 @@ static const char * const qp_s[] = { "none", "init", "start", "stop", "close", }; +static const u32 typical_qos_val[QM_QOS_TYPICAL_NUM] = {100, 250, 500, 1000, + 10000, 25000, 50000, 100000}; +static const u32 typical_qos_cbs_s[QM_QOS_TYPICAL_NUM] = {9, 10, 11, 12, 16, + 17, 18, 19}; + static bool qm_avail_state(struct hisi_qm *qm, enum qm_state new) { enum qm_state curr = atomic_read(&qm->status.flags); @@ -491,6 +581,18 @@ static bool qm_qp_avail_state(struct hisi_qm *qm, struct hisi_qp *qp, return avail; } +static void qm_mb_pre_init(struct qm_mailbox *mailbox, u8 cmd, + u64 base, u16 queue, bool op) +{ + mailbox->w0 = cpu_to_le16((cmd) | + ((op) ? 0x1 << QM_MB_OP_SHIFT : 0) | + (0x1 << QM_MB_BUSY_SHIFT)); + mailbox->queue_num = cpu_to_le16(queue); + mailbox->base_l = cpu_to_le32(lower_32_bits(base)); + mailbox->base_h = cpu_to_le32(upper_32_bits(base)); + mailbox->rsvd = 0; +} + /* return 0 mailbox ready, -ETIMEDOUT hardware timeout */ static int qm_wait_mb_ready(struct hisi_qm *qm) { @@ -523,44 +625,42 @@ static void qm_mb_write(struct hisi_qm *qm, const void *src) : "memory"); } -static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, - bool op) +static int qm_mb_nolock(struct hisi_qm *qm, struct qm_mailbox *mailbox) { - struct qm_mailbox mailbox; - int ret = 0; - - dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n", - queue, cmd, (unsigned long long)dma_addr); - - mailbox.w0 = cpu_to_le16(cmd | - (op ? 0x1 << QM_MB_OP_SHIFT : 0) | - (0x1 << QM_MB_BUSY_SHIFT)); - mailbox.queue_num = cpu_to_le16(queue); - mailbox.base_l = cpu_to_le32(lower_32_bits(dma_addr)); - mailbox.base_h = cpu_to_le32(upper_32_bits(dma_addr)); - mailbox.rsvd = 0; - - mutex_lock(&qm->mailbox_lock); - if (unlikely(qm_wait_mb_ready(qm))) { - ret = -EBUSY; dev_err(&qm->pdev->dev, "QM mailbox is busy to start!\n"); - goto busy_unlock; + goto mb_busy; } - qm_mb_write(qm, &mailbox); + qm_mb_write(qm, mailbox); if (unlikely(qm_wait_mb_ready(qm))) { - ret = -EBUSY; dev_err(&qm->pdev->dev, "QM mailbox operation timeout!\n"); - goto busy_unlock; + goto mb_busy; } -busy_unlock: + return 0; + +mb_busy: + atomic64_inc(&qm->debug.dfx.mb_err_cnt); + return -EBUSY; +} + +static int qm_mb(struct hisi_qm *qm, u8 cmd, dma_addr_t dma_addr, u16 queue, + bool op) +{ + struct qm_mailbox mailbox; + int ret; + + dev_dbg(&qm->pdev->dev, "QM mailbox request to q%u: %u-%llx\n", + queue, cmd, (unsigned long long)dma_addr); + + qm_mb_pre_init(&mailbox, cmd, dma_addr, queue, op); + + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox); mutex_unlock(&qm->mailbox_lock); - if (ret) - atomic64_inc(&qm->debug.dfx.mb_err_cnt); return ret; } @@ -626,6 +726,14 @@ static u32 qm_get_irq_num_v2(struct hisi_qm *qm) return QM_IRQ_NUM_VF_V2; } +static u32 qm_get_irq_num_v3(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_PF) + return QM_IRQ_NUM_PF_V2; + + return QM_IRQ_NUM_VF_V3; +} + static struct hisi_qp *qm_to_hisi_qp(struct hisi_qm *qm, struct qm_eqe *eqe) { u16 cqn = le32_to_cpu(eqe->dw0) & QM_EQE_CQN_MASK; @@ -730,6 +838,21 @@ static irqreturn_t qm_irq(int irq, void *data) return IRQ_NONE; } +static irqreturn_t qm_mb_cmd_irq(int irq, void *data) +{ + struct hisi_qm *qm = data; + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_STATUS); + val &= QM_IFC_INT_STATUS_MASK; + if (!val) + return IRQ_NONE; + + schedule_work(&qm->cmd_process); + + return IRQ_HANDLED; +} + static irqreturn_t qm_aeq_irq(int irq, void *data) { struct hisi_qm *qm = data; @@ -770,14 +893,16 @@ static void qm_irq_unregister(struct hisi_qm *qm) free_irq(pci_irq_vector(pdev, QM_EQ_EVENT_IRQ_VECTOR), qm); - if (qm->ver == QM_HW_V1) - return; + if (qm->ver > QM_HW_V1) { + free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); - free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); + if (qm->fun_type == QM_HW_PF) + free_irq(pci_irq_vector(pdev, + QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); + } - if (qm->fun_type == QM_HW_PF) - free_irq(pci_irq_vector(pdev, - QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); + if (qm->ver > QM_HW_V2) + free_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), qm); } static void qm_init_qp_status(struct hisi_qp *qp) @@ -790,8 +915,95 @@ static void qm_init_qp_status(struct hisi_qp *qp) atomic_set(&qp_status->used, 0); } +static void qm_init_prefetch(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 page_type = 0x0; + + if (qm->ver < QM_HW_V3) + return; + + switch (PAGE_SIZE) { + case SZ_4K: + page_type = 0x0; + break; + case SZ_16K: + page_type = 0x1; + break; + case SZ_64K: + page_type = 0x2; + break; + default: + dev_err(dev, "system page size is not support: %lu, default set to 4KB", + PAGE_SIZE); + } + + writel(page_type, qm->io_base + QM_PAGE_SIZE); +} + +/* + * the formula: + * IR = X Mbps if ir = 1 means IR = 100 Mbps, if ir = 10000 means = 10Gbps + * + * IR_b * (2 ^ IR_u) * 8 + * IR(Mbps) * 10 ^ -3 = ------------------------- + * Tick * (2 ^ IR_s) + */ +static u32 acc_shaper_para_calc(u64 cir_b, u64 cir_u, u64 cir_s) +{ + return ((cir_b * QM_QOS_DIVISOR_CLK) * (1 << cir_u)) / + (QM_QOS_TICK * (1 << cir_s)); +} + +static u32 acc_shaper_calc_cbs_s(u32 ir) +{ + int i; + + if (ir < typical_qos_val[0]) + return QM_SHAPER_MIN_CBS_S; + + for (i = 1; i < QM_QOS_TYPICAL_NUM; i++) { + if (ir >= typical_qos_val[i - 1] && ir < typical_qos_val[i]) + return typical_qos_cbs_s[i - 1]; + } + + return typical_qos_cbs_s[QM_QOS_TYPICAL_NUM - 1]; +} + +static int qm_get_shaper_para(u32 ir, struct qm_shaper_factor *factor) +{ + u32 cir_b, cir_u, cir_s, ir_calc; + u32 error_rate; + + factor->cbs_s = acc_shaper_calc_cbs_s(ir); + + for (cir_b = QM_QOS_MIN_CIR_B; cir_b <= QM_QOS_MAX_CIR_B; cir_b++) { + for (cir_u = 0; cir_u <= QM_QOS_MAX_CIR_U; cir_u++) { + for (cir_s = 0; cir_s <= QM_QOS_MAX_CIR_S; cir_s++) { + /** the formula is changed to: + * IR_b * (2 ^ IR_u) * DIVISOR_CLK + * IR(Mbps) = ------------------------- + * 768 * (2 ^ IR_s) + */ + ir_calc = acc_shaper_para_calc(cir_b, cir_u, + cir_s); + error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; + if (error_rate <= QM_QOS_MIN_ERROR_RATE) { + factor->cir_b = cir_b; + factor->cir_u = cir_u; + factor->cir_s = cir_s; + + return 0; + } + } + } + } + + return -EINVAL; +} + static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, - u32 number) + u32 number, struct qm_shaper_factor *factor) { u64 tmp = 0; @@ -820,6 +1032,15 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, tmp = QM_CQC_VFT_VALID; } break; + case SHAPER_VFT: + if (qm->ver >= QM_HW_V3) { + tmp = factor->cir_b | + (factor->cir_u << QM_SHAPER_FACTOR_CIR_U_SHIFT) | + (factor->cir_s << QM_SHAPER_FACTOR_CIR_S_SHIFT) | + (QM_SHAPER_CBS_B << QM_SHAPER_FACTOR_CBS_B_SHIFT) | + (factor->cbs_s << QM_SHAPER_FACTOR_CBS_S_SHIFT); + } + break; } } @@ -830,6 +1051,7 @@ static void qm_vft_data_cfg(struct hisi_qm *qm, enum vft_type type, u32 base, static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, u32 fun_num, u32 base, u32 number) { + struct qm_shaper_factor *factor = &qm->factor[fun_num]; unsigned int val; int ret; @@ -841,9 +1063,12 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, writel(0x0, qm->io_base + QM_VFT_CFG_OP_WR); writel(type, qm->io_base + QM_VFT_CFG_TYPE); + if (type == SHAPER_VFT) + fun_num |= base << QM_SHAPER_VFT_OFFSET; + writel(fun_num, qm->io_base + QM_VFT_CFG); - qm_vft_data_cfg(qm, type, base, number); + qm_vft_data_cfg(qm, type, base, number, factor); writel(0x0, qm->io_base + QM_VFT_CFG_RDY); writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); @@ -853,6 +1078,27 @@ static int qm_set_vft_common(struct hisi_qm *qm, enum vft_type type, POLL_TIMEOUT); } +static int qm_shaper_init_vft(struct hisi_qm *qm, u32 fun_num) +{ + int ret, i; + + qm->factor[fun_num].func_qos = QM_QOS_MAX_VAL; + ret = qm_get_shaper_para(QM_QOS_MAX_VAL * QM_QOS_RATE, &qm->factor[fun_num]); + if (ret) { + dev_err(&qm->pdev->dev, "failed to calculate shaper parameter!\n"); + return ret; + } + writel(qm->type_rate, qm->io_base + QM_SHAPER_CFG); + for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { + /* The base number of queue reuse for different alg type */ + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_num, i, 1); + if (ret) + return ret; + } + + return 0; +} + /* The config should be conducted after qm_dev_mem_reset() */ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, u32 number) @@ -865,7 +1111,21 @@ static int qm_set_sqc_cqc_vft(struct hisi_qm *qm, u32 fun_num, u32 base, return ret; } + /* init default shaper qos val */ + if (qm->ver >= QM_HW_V3) { + ret = qm_shaper_init_vft(qm, fun_num); + if (ret) + goto back_sqc_cqc; + } + return 0; +back_sqc_cqc: + for (i = SQC_VFT; i <= CQC_VFT; i++) { + ret = qm_set_vft_common(qm, i, fun_num, 0, 0); + if (ret) + return ret; + } + return ret; } static int qm_get_vft_v2(struct hisi_qm *qm, u32 *base, u32 *number) @@ -1570,16 +1830,9 @@ static ssize_t qm_cmd_write(struct file *filp, const char __user *buffer, if (count > QM_DBG_WRITE_LEN) return -ENOSPC; - cmd_buf = kzalloc(count + 1, GFP_KERNEL); - if (!cmd_buf) - return -ENOMEM; - - if (copy_from_user(cmd_buf, buffer, count)) { - kfree(cmd_buf); - return -EFAULT; - } - - cmd_buf[count] = '\0'; + cmd_buf = memdup_user_nul(buffer, count); + if (IS_ERR(cmd_buf)) + return PTR_ERR(cmd_buf); cmd_buf_tmp = strchr(cmd_buf, '\n'); if (cmd_buf_tmp) { @@ -1623,13 +1876,9 @@ static void qm_hw_error_init_v1(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); } -static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +static void qm_hw_error_cfg(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) { - u32 irq_enable = ce | nfe | fe; - u32 irq_unmask = ~irq_enable; - qm->error_mask = ce | nfe | fe; - /* clear QM hw residual error source */ writel(QM_ABNORMAL_INT_SOURCE_CLR, qm->io_base + QM_ABNORMAL_INT_SOURCE); @@ -1639,6 +1888,14 @@ static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) writel(QM_RAS_CE_TIMES_PER_IRQ, qm->io_base + QM_RAS_CE_THRESHOLD); writel(nfe, qm->io_base + QM_RAS_NFE_ENABLE); writel(fe, qm->io_base + QM_RAS_FE_ENABLE); +} + +static void qm_hw_error_init_v2(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +{ + u32 irq_enable = ce | nfe | fe; + u32 irq_unmask = ~irq_enable; + + qm_hw_error_cfg(qm, ce, nfe, fe); irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); @@ -1649,6 +1906,28 @@ static void qm_hw_error_uninit_v2(struct hisi_qm *qm) writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); } +static void qm_hw_error_init_v3(struct hisi_qm *qm, u32 ce, u32 nfe, u32 fe) +{ + u32 irq_enable = ce | nfe | fe; + u32 irq_unmask = ~irq_enable; + + qm_hw_error_cfg(qm, ce, nfe, fe); + + /* enable close master ooo when hardware error happened */ + writel(nfe & (~QM_DB_RANDOM_INVALID), qm->io_base + QM_OOO_SHUTDOWN_SEL); + + irq_unmask &= readl(qm->io_base + QM_ABNORMAL_INT_MASK); + writel(irq_unmask, qm->io_base + QM_ABNORMAL_INT_MASK); +} + +static void qm_hw_error_uninit_v3(struct hisi_qm *qm) +{ + writel(QM_ABNORMAL_INT_MASK_VALUE, qm->io_base + QM_ABNORMAL_INT_MASK); + + /* disable close master ooo when hardware error happened */ + writel(0x0, qm->io_base + QM_OOO_SHUTDOWN_SEL); +} + static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) { const struct hisi_qm_hw_error *err; @@ -1715,15 +1994,371 @@ static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) return ACC_ERR_RECOVERED; } +static u32 qm_get_hw_error_status(struct hisi_qm *qm) +{ + return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); +} + +static u32 qm_get_dev_err_status(struct hisi_qm *qm) +{ + return qm->err_ini->get_dev_hw_err_status(qm); +} + +/* Check if the error causes the master ooo block */ +static int qm_check_dev_error(struct hisi_qm *qm) +{ + u32 val, dev_val; + + if (qm->fun_type == QM_HW_VF) + return 0; + + val = qm_get_hw_error_status(qm); + dev_val = qm_get_dev_err_status(qm); + + if (qm->ver < QM_HW_V3) + return (val & QM_ECC_MBIT) || + (dev_val & qm->err_info.ecc_2bits_mask); + + return (val & readl(qm->io_base + QM_OOO_SHUTDOWN_SEL)) || + (dev_val & (~qm->err_info.dev_ce_mask)); +} + +static int qm_get_mb_cmd(struct hisi_qm *qm, u64 *msg, u16 fun_num) +{ + struct qm_mailbox mailbox; + int ret; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_DST, 0, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox); + if (ret) + goto err_unlock; + + *msg = readl(qm->io_base + QM_MB_CMD_DATA_ADDR_L) | + ((u64)readl(qm->io_base + QM_MB_CMD_DATA_ADDR_H) << 32); + +err_unlock: + mutex_unlock(&qm->mailbox_lock); + return ret; +} + +static void qm_clear_cmd_interrupt(struct hisi_qm *qm, u64 vf_mask) +{ + u32 val; + + if (qm->fun_type == QM_HW_PF) + writeq(vf_mask, qm->io_base + QM_IFC_INT_SOURCE_P); + + val = readl(qm->io_base + QM_IFC_INT_SOURCE_V); + val |= QM_IFC_INT_SOURCE_MASK; + writel(val, qm->io_base + QM_IFC_INT_SOURCE_V); +} + +static void qm_handle_vf_msg(struct hisi_qm *qm, u32 vf_id) +{ + struct device *dev = &qm->pdev->dev; + u32 cmd; + u64 msg; + int ret; + + ret = qm_get_mb_cmd(qm, &msg, vf_id); + if (ret) { + dev_err(dev, "failed to get msg from VF(%u)!\n", vf_id); + return; + } + + cmd = msg & QM_MB_CMD_DATA_MASK; + switch (cmd) { + case QM_VF_PREPARE_FAIL: + dev_err(dev, "failed to stop VF(%u)!\n", vf_id); + break; + case QM_VF_START_FAIL: + dev_err(dev, "failed to start VF(%u)!\n", vf_id); + break; + case QM_VF_PREPARE_DONE: + case QM_VF_START_DONE: + break; + default: + dev_err(dev, "unsupported cmd %u sent by VF(%u)!\n", cmd, vf_id); + break; + } +} + +static int qm_wait_vf_prepare_finish(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 vfs_num = qm->vfs_num; + int cnt = 0; + int ret = 0; + u64 val; + u32 i; + + if (!qm->vfs_num || qm->ver < QM_HW_V3) + return 0; + + while (true) { + val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); + /* All VFs send command to PF, break */ + if ((val & GENMASK(vfs_num, 1)) == GENMASK(vfs_num, 1)) + break; + + if (++cnt > QM_MAX_PF_WAIT_COUNT) { + ret = -EBUSY; + break; + } + + msleep(QM_WAIT_DST_ACK); + } + + /* PF check VFs msg */ + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + qm_handle_vf_msg(qm, i); + else + dev_err(dev, "VF(%u) not ping PF!\n", i); + } + + /* PF clear interrupt to ack VFs */ + qm_clear_cmd_interrupt(qm, val); + + return ret; +} + +static void qm_trigger_vf_interrupt(struct hisi_qm *qm, u32 fun_num) +{ + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_CFG); + val &= ~QM_IFC_SEND_ALL_VFS; + val |= fun_num; + writel(val, qm->io_base + QM_IFC_INT_CFG); + + val = readl(qm->io_base + QM_IFC_INT_SET_P); + val |= QM_IFC_INT_SET_MASK; + writel(val, qm->io_base + QM_IFC_INT_SET_P); +} + +static void qm_trigger_pf_interrupt(struct hisi_qm *qm) +{ + u32 val; + + val = readl(qm->io_base + QM_IFC_INT_SET_V); + val |= QM_IFC_INT_SET_MASK; + writel(val, qm->io_base + QM_IFC_INT_SET_V); +} + +static int qm_ping_single_vf(struct hisi_qm *qm, u64 cmd, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + struct qm_mailbox mailbox; + int cnt = 0; + u64 val; + int ret; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, fun_num, 0); + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox); + if (ret) { + dev_err(dev, "failed to send command to vf(%u)!\n", fun_num); + goto err_unlock; + } + + qm_trigger_vf_interrupt(qm, fun_num); + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readq(qm->io_base + QM_IFC_READY_STATUS); + /* if VF respond, PF notifies VF successfully. */ + if (!(val & BIT(fun_num))) + goto err_unlock; + + if (++cnt > QM_MAX_PF_WAIT_COUNT) { + dev_err(dev, "failed to get response from VF(%u)!\n", fun_num); + ret = -ETIMEDOUT; + break; + } + } + +err_unlock: + mutex_unlock(&qm->mailbox_lock); + return ret; +} + +static int qm_ping_all_vfs(struct hisi_qm *qm, u64 cmd) +{ + struct device *dev = &qm->pdev->dev; + u32 vfs_num = qm->vfs_num; + struct qm_mailbox mailbox; + u64 val = 0; + int cnt = 0; + int ret; + u32 i; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, QM_MB_PING_ALL_VFS, 0); + mutex_lock(&qm->mailbox_lock); + /* PF sends command to all VFs by mailbox */ + ret = qm_mb_nolock(qm, &mailbox); + if (ret) { + dev_err(dev, "failed to send command to VFs!\n"); + mutex_unlock(&qm->mailbox_lock); + return ret; + } + + qm_trigger_vf_interrupt(qm, QM_IFC_SEND_ALL_VFS); + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readq(qm->io_base + QM_IFC_READY_STATUS); + /* If all VFs acked, PF notifies VFs successfully. */ + if (!(val & GENMASK(vfs_num, 1))) { + mutex_unlock(&qm->mailbox_lock); + return 0; + } + + if (++cnt > QM_MAX_PF_WAIT_COUNT) + break; + } + + mutex_unlock(&qm->mailbox_lock); + + /* Check which vf respond timeout. */ + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + dev_err(dev, "failed to get response from VF(%u)!\n", i); + } + + return -ETIMEDOUT; +} + +static int qm_ping_pf(struct hisi_qm *qm, u64 cmd) +{ + struct qm_mailbox mailbox; + int cnt = 0; + u32 val; + int ret; + + qm_mb_pre_init(&mailbox, QM_MB_CMD_SRC, cmd, 0, 0); + mutex_lock(&qm->mailbox_lock); + ret = qm_mb_nolock(qm, &mailbox); + if (ret) { + dev_err(&qm->pdev->dev, "failed to send command to PF!\n"); + goto unlock; + } + + qm_trigger_pf_interrupt(qm); + /* Waiting for PF response */ + while (true) { + msleep(QM_WAIT_DST_ACK); + val = readl(qm->io_base + QM_IFC_INT_SET_V); + if (!(val & QM_IFC_INT_STATUS_MASK)) + break; + + if (++cnt > QM_MAX_VF_WAIT_COUNT) { + ret = -ETIMEDOUT; + break; + } + } + +unlock: + mutex_unlock(&qm->mailbox_lock); + return ret; +} + static int qm_stop_qp(struct hisi_qp *qp) { return qm_mb(qp->qm, QM_MB_CMD_STOP_QP, 0, qp->qp_id, 0); } +static int qm_set_msi(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + + if (set) { + pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, + 0); + } else { + pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, + ACC_PEH_MSI_DISABLE); + if (qm->err_status.is_qm_ecc_mbit || + qm->err_status.is_dev_ecc_mbit) + return 0; + + mdelay(1); + if (readl(qm->io_base + QM_PEH_DFX_INFO0)) + return -EFAULT; + } + + return 0; +} + +static void qm_wait_msi_finish(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + u32 cmd = ~0; + int cnt = 0; + u32 val; + int ret; + + while (true) { + pci_read_config_dword(pdev, pdev->msi_cap + + PCI_MSI_PENDING_64, &cmd); + if (!cmd) + break; + + if (++cnt > MAX_WAIT_COUNTS) { + pci_warn(pdev, "failed to empty MSI PENDING!\n"); + break; + } + + udelay(1); + } + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO0, + val, !(val & QM_PEH_DFX_MASK), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + pci_warn(pdev, "failed to empty PEH MSI!\n"); + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_DFX_INFO1, + val, !(val & QM_PEH_MSI_FINISH_MASK), + POLL_PERIOD, POLL_TIMEOUT); + if (ret) + pci_warn(pdev, "failed to finish MSI operation!\n"); +} + +static int qm_set_msi_v3(struct hisi_qm *qm, bool set) +{ + struct pci_dev *pdev = qm->pdev; + int ret = -ETIMEDOUT; + u32 cmd, i; + + pci_read_config_dword(pdev, pdev->msi_cap, &cmd); + if (set) + cmd |= QM_MSI_CAP_ENABLE; + else + cmd &= ~QM_MSI_CAP_ENABLE; + + pci_write_config_dword(pdev, pdev->msi_cap, cmd); + if (set) { + for (i = 0; i < MAX_WAIT_COUNTS; i++) { + pci_read_config_dword(pdev, pdev->msi_cap, &cmd); + if (cmd & QM_MSI_CAP_ENABLE) + return 0; + + udelay(1); + } + } else { + udelay(WAIT_PERIOD_US_MIN); + qm_wait_msi_finish(qm); + ret = 0; + } + + return ret; +} + static const struct hisi_qm_hw_ops qm_hw_ops_v1 = { .qm_db = qm_db_v1, .get_irq_num = qm_get_irq_num_v1, .hw_error_init = qm_hw_error_init_v1, + .set_msi = qm_set_msi, }; static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { @@ -1733,16 +2368,20 @@ static const struct hisi_qm_hw_ops qm_hw_ops_v2 = { .hw_error_init = qm_hw_error_init_v2, .hw_error_uninit = qm_hw_error_uninit_v2, .hw_error_handle = qm_hw_error_handle_v2, + .set_msi = qm_set_msi, }; static const struct hisi_qm_hw_ops qm_hw_ops_v3 = { .get_vft = qm_get_vft_v2, .qm_db = qm_db_v2, - .get_irq_num = qm_get_irq_num_v2, - .hw_error_init = qm_hw_error_init_v2, - .hw_error_uninit = qm_hw_error_uninit_v2, + .get_irq_num = qm_get_irq_num_v3, + .hw_error_init = qm_hw_error_init_v3, + .hw_error_uninit = qm_hw_error_uninit_v3, .hw_error_handle = qm_hw_error_handle_v2, .stop_qp = qm_stop_qp, + .set_msi = qm_set_msi_v3, + .ping_all_vfs = qm_ping_all_vfs, + .ping_pf = qm_ping_pf, }; static void *qm_get_avail_sqe(struct hisi_qp *qp) @@ -2017,11 +2656,8 @@ static int qm_drain_qp(struct hisi_qp *qp) int ret = 0, i = 0; void *addr; - /* - * No need to judge if ECC multi-bit error occurs because the - * master OOO will be blocked. - */ - if (qm->err_status.is_qm_ecc_mbit || qm->err_status.is_dev_ecc_mbit) + /* No need to judge if master OOO is blocked. */ + if (qm_check_dev_error(qm)) return 0; /* Kunpeng930 supports drain qp by device */ @@ -2290,6 +2926,23 @@ static void hisi_qm_uacce_stop_queue(struct uacce_queue *q) hisi_qm_stop_qp(q->priv); } +static int hisi_qm_is_q_updated(struct uacce_queue *q) +{ + struct hisi_qp *qp = q->priv; + struct qm_cqe *cqe = qp->cqe + qp->qp_status.cq_head; + int updated = 0; + + while (QM_CQE_PHASE(cqe) == qp->qp_status.cqc_phase) { + /* make sure to read data from memory */ + dma_rmb(); + qm_cq_head_update(qp); + cqe = qp->cqe + qp->qp_status.cq_head; + updated = 1; + } + + return updated; +} + static void qm_set_sqctype(struct uacce_queue *q, u16 type) { struct hisi_qm *qm = q->uacce->priv; @@ -2335,6 +2988,7 @@ static const struct uacce_ops uacce_qm_ops = { .stop_queue = hisi_qm_uacce_stop_queue, .mmap = hisi_qm_uacce_mmap, .ioctl = hisi_qm_uacce_ioctl, + .is_q_updated = hisi_qm_is_q_updated, }; static int qm_alloc_uacce(struct hisi_qm *qm) @@ -2530,62 +3184,6 @@ static int hisi_qp_memory_init(struct hisi_qm *qm, size_t dma_size, int id) return 0; } -static int hisi_qm_memory_init(struct hisi_qm *qm) -{ - struct device *dev = &qm->pdev->dev; - size_t qp_dma_size, off = 0; - int i, ret = 0; - -#define QM_INIT_BUF(qm, type, num) do { \ - (qm)->type = ((qm)->qdma.va + (off)); \ - (qm)->type##_dma = (qm)->qdma.dma + (off); \ - off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ -} while (0) - - idr_init(&qm->qp_idr); - qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + - QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + - QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + - QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); - qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, - GFP_ATOMIC); - dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); - if (!qm->qdma.va) - return -ENOMEM; - - QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); - QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); - QM_INIT_BUF(qm, sqc, qm->qp_num); - QM_INIT_BUF(qm, cqc, qm->qp_num); - - qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); - if (!qm->qp_array) { - ret = -ENOMEM; - goto err_alloc_qp_array; - } - - /* one more page for device or qp statuses */ - qp_dma_size = qm->sqe_size * QM_Q_DEPTH + - sizeof(struct qm_cqe) * QM_Q_DEPTH; - qp_dma_size = PAGE_ALIGN(qp_dma_size); - for (i = 0; i < qm->qp_num; i++) { - ret = hisi_qp_memory_init(qm, qp_dma_size, i); - if (ret) - goto err_init_qp_mem; - - dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); - } - - return ret; - -err_init_qp_mem: - hisi_qp_memory_uninit(qm, i); -err_alloc_qp_array: - dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); - - return ret; -} - static void hisi_qm_pre_init(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -2604,6 +3202,34 @@ static void hisi_qm_pre_init(struct hisi_qm *qm) qm->misc_ctl = false; } +static void qm_cmd_uninit(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + val = readl(qm->io_base + QM_IFC_INT_MASK); + val |= QM_IFC_INT_DISABLE; + writel(val, qm->io_base + QM_IFC_INT_MASK); +} + +static void qm_cmd_init(struct hisi_qm *qm) +{ + u32 val; + + if (qm->ver < QM_HW_V3) + return; + + /* Clear communication interrupt source */ + qm_clear_cmd_interrupt(qm, QM_IFC_INT_SOURCE_CLR); + + /* Enable pf to vf communication reg. */ + val = readl(qm->io_base + QM_IFC_INT_MASK); + val &= ~QM_IFC_INT_DISABLE; + writel(val, qm->io_base + QM_IFC_INT_MASK); +} + static void qm_put_pci_res(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -2635,6 +3261,8 @@ void hisi_qm_uninit(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; struct device *dev = &pdev->dev; + qm_cmd_uninit(qm); + kfree(qm->factor); down_write(&qm->qps_lock); if (!qm_avail_state(qm, QM_CLOSE)) { @@ -2826,6 +3454,8 @@ static int __hisi_qm_start(struct hisi_qm *qm) if (ret) return ret; + qm_init_prefetch(qm); + writel(0x0, qm->io_base + QM_VF_EQ_INT_MASK); writel(0x0, qm->io_base + QM_VF_AEQ_INT_MASK); @@ -3034,79 +3664,6 @@ static int qm_debugfs_atomic64_get(void *data, u64 *val) DEFINE_DEBUGFS_ATTRIBUTE(qm_atomic64_ops, qm_debugfs_atomic64_get, qm_debugfs_atomic64_set, "%llu\n"); -/** - * hisi_qm_debug_init() - Initialize qm related debugfs files. - * @qm: The qm for which we want to add debugfs files. - * - * Create qm related debugfs files. - */ -void hisi_qm_debug_init(struct hisi_qm *qm) -{ - struct qm_dfx *dfx = &qm->debug.dfx; - struct dentry *qm_d; - void *data; - int i; - - qm_d = debugfs_create_dir("qm", qm->debug.debug_root); - qm->debug.qm_d = qm_d; - - /* only show this in PF */ - if (qm->fun_type == QM_HW_PF) { - qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM); - for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) - qm_create_debugfs_file(qm, qm_d, i); - } - - debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); - - debugfs_create_file("cmd", 0444, qm->debug.qm_d, qm, &qm_cmd_fops); - - debugfs_create_file("status", 0444, qm->debug.qm_d, qm, - &qm_status_fops); - for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) { - data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset); - debugfs_create_file(qm_dfx_files[i].name, - 0644, - qm_d, - data, - &qm_atomic64_ops); - } -} -EXPORT_SYMBOL_GPL(hisi_qm_debug_init); - -/** - * hisi_qm_debug_regs_clear() - clear qm debug related registers. - * @qm: The qm for which we want to clear its debug registers. - */ -void hisi_qm_debug_regs_clear(struct hisi_qm *qm) -{ - struct qm_dfx_registers *regs; - int i; - - /* clear current_qm */ - writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); - writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); - - /* clear current_q */ - writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); - writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); - - /* - * these registers are reading and clearing, so clear them after - * reading them. - */ - writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE); - - regs = qm_dfx_regs; - for (i = 0; i < CNT_CYC_REGS_NUM; i++) { - readl(qm->io_base + regs->reg_offset); - regs++; - } - - writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE); -} -EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); - static void qm_hw_error_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -3362,6 +3919,360 @@ static int qm_clear_vft_config(struct hisi_qm *qm) return 0; } +static int qm_func_shaper_enable(struct hisi_qm *qm, u32 fun_index, u32 qos) +{ + struct device *dev = &qm->pdev->dev; + u32 ir = qos * QM_QOS_RATE; + int ret, total_vfs, i; + + total_vfs = pci_sriov_get_totalvfs(qm->pdev); + if (fun_index > total_vfs) + return -EINVAL; + + qm->factor[fun_index].func_qos = qos; + + ret = qm_get_shaper_para(ir, &qm->factor[fun_index]); + if (ret) { + dev_err(dev, "failed to calculate shaper parameter!\n"); + return -EINVAL; + } + + for (i = ALG_TYPE_0; i <= ALG_TYPE_1; i++) { + /* The base number of queue reuse for different alg type */ + ret = qm_set_vft_common(qm, SHAPER_VFT, fun_index, i, 1); + if (ret) { + dev_err(dev, "type: %d, failed to set shaper vft!\n", i); + return -EINVAL; + } + } + + return 0; +} + +static u32 qm_get_shaper_vft_qos(struct hisi_qm *qm, u32 fun_index) +{ + u64 cir_u = 0, cir_b = 0, cir_s = 0; + u64 shaper_vft, ir_calc, ir; + unsigned int val; + u32 error_rate; + int ret; + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); + if (ret) + return 0; + + writel(0x1, qm->io_base + QM_VFT_CFG_OP_WR); + writel(SHAPER_VFT, qm->io_base + QM_VFT_CFG_TYPE); + writel(fun_index, qm->io_base + QM_VFT_CFG); + + writel(0x0, qm->io_base + QM_VFT_CFG_RDY); + writel(0x1, qm->io_base + QM_VFT_CFG_OP_ENABLE); + + ret = readl_relaxed_poll_timeout(qm->io_base + QM_VFT_CFG_RDY, val, + val & BIT(0), POLL_PERIOD, + POLL_TIMEOUT); + if (ret) + return 0; + + shaper_vft = readl(qm->io_base + QM_VFT_CFG_DATA_L) | + ((u64)readl(qm->io_base + QM_VFT_CFG_DATA_H) << 32); + + cir_b = shaper_vft & QM_SHAPER_CIR_B_MASK; + cir_u = shaper_vft & QM_SHAPER_CIR_U_MASK; + cir_u = cir_u >> QM_SHAPER_FACTOR_CIR_U_SHIFT; + + cir_s = shaper_vft & QM_SHAPER_CIR_S_MASK; + cir_s = cir_s >> QM_SHAPER_FACTOR_CIR_S_SHIFT; + + ir_calc = acc_shaper_para_calc(cir_b, cir_u, cir_s); + + ir = qm->factor[fun_index].func_qos * QM_QOS_RATE; + + error_rate = QM_QOS_EXPAND_RATE * (u32)abs(ir_calc - ir) / ir; + if (error_rate > QM_QOS_MIN_ERROR_RATE) { + pci_err(qm->pdev, "error_rate: %u, get function qos is error!\n", error_rate); + return 0; + } + + return ir; +} + +static void qm_vf_get_qos(struct hisi_qm *qm, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + u64 mb_cmd; + u32 qos; + int ret; + + qos = qm_get_shaper_vft_qos(qm, fun_num); + if (!qos) { + dev_err(dev, "function(%u) failed to get qos by PF!\n", fun_num); + return; + } + + mb_cmd = QM_PF_SET_QOS | (u64)qos << QM_MB_CMD_DATA_SHIFT; + ret = qm_ping_single_vf(qm, mb_cmd, fun_num); + if (ret) + dev_err(dev, "failed to send cmd to VF(%u)!\n", fun_num); +} + +static int qm_vf_read_qos(struct hisi_qm *qm) +{ + int cnt = 0; + int ret; + + /* reset mailbox qos val */ + qm->mb_qos = 0; + + /* vf ping pf to get function qos */ + if (qm->ops->ping_pf) { + ret = qm->ops->ping_pf(qm, QM_VF_GET_QOS); + if (ret) { + pci_err(qm->pdev, "failed to send cmd to PF to get qos!\n"); + return ret; + } + } + + while (true) { + msleep(QM_WAIT_DST_ACK); + if (qm->mb_qos) + break; + + if (++cnt > QM_MAX_VF_WAIT_COUNT) { + pci_err(qm->pdev, "PF ping VF timeout!\n"); + return -ETIMEDOUT; + } + } + + return ret; +} + +static ssize_t qm_algqos_read(struct file *filp, char __user *buf, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char tbuf[QM_DBG_READ_LEN]; + u32 qos_val, ir; + int ret; + + /* Mailbox and reset cannot be operated at the same time */ + if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + pci_err(qm->pdev, "dev resetting, read alg qos failed!\n"); + return -EAGAIN; + } + + if (qm->fun_type == QM_HW_PF) { + ir = qm_get_shaper_vft_qos(qm, 0); + } else { + ret = qm_vf_read_qos(qm); + if (ret) + goto err_get_status; + ir = qm->mb_qos; + } + + qos_val = ir / QM_QOS_RATE; + ret = scnprintf(tbuf, QM_DBG_READ_LEN, "%u\n", qos_val); + + ret = simple_read_from_buffer(buf, count, pos, tbuf, ret); + +err_get_status: + clear_bit(QM_RESETTING, &qm->misc_ctl); + return ret; +} + +static ssize_t qm_qos_value_init(const char *buf, unsigned long *val) +{ + int buflen = strlen(buf); + int ret, i; + + for (i = 0; i < buflen; i++) { + if (!isdigit(buf[i])) + return -EINVAL; + } + + ret = sscanf(buf, "%ld", val); + if (ret != QM_QOS_VAL_NUM) + return -EINVAL; + + return 0; +} + +static ssize_t qm_algqos_write(struct file *filp, const char __user *buf, + size_t count, loff_t *pos) +{ + struct hisi_qm *qm = filp->private_data; + char tbuf[QM_DBG_READ_LEN]; + int tmp1, bus, device, function; + char tbuf_bdf[QM_DBG_READ_LEN] = {0}; + char val_buf[QM_QOS_VAL_MAX_LEN] = {0}; + unsigned int fun_index; + unsigned long val = 0; + int len, ret; + + if (qm->fun_type == QM_HW_VF) + return -EINVAL; + + /* Mailbox and reset cannot be operated at the same time */ + if (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { + pci_err(qm->pdev, "dev resetting, write alg qos failed!\n"); + return -EAGAIN; + } + + if (*pos != 0) { + ret = 0; + goto err_get_status; + } + + if (count >= QM_DBG_READ_LEN) { + ret = -ENOSPC; + goto err_get_status; + } + + len = simple_write_to_buffer(tbuf, QM_DBG_READ_LEN - 1, pos, buf, count); + if (len < 0) { + ret = len; + goto err_get_status; + } + + tbuf[len] = '\0'; + ret = sscanf(tbuf, "%s %s", tbuf_bdf, val_buf); + if (ret != QM_QOS_PARAM_NUM) { + ret = -EINVAL; + goto err_get_status; + } + + ret = qm_qos_value_init(val_buf, &val); + if (val == 0 || val > QM_QOS_MAX_VAL || ret) { + pci_err(qm->pdev, "input qos value is error, please set 1~1000!\n"); + ret = -EINVAL; + goto err_get_status; + } + + ret = sscanf(tbuf_bdf, "%d:%x:%d.%d", &tmp1, &bus, &device, &function); + if (ret != QM_QOS_BDF_PARAM_NUM) { + pci_err(qm->pdev, "input pci bdf value is error!\n"); + ret = -EINVAL; + goto err_get_status; + } + + fun_index = device * 8 + function; + + ret = qm_func_shaper_enable(qm, fun_index, val); + if (ret) { + pci_err(qm->pdev, "failed to enable function shaper!\n"); + ret = -EINVAL; + goto err_get_status; + } + + ret = count; + +err_get_status: + clear_bit(QM_RESETTING, &qm->misc_ctl); + return ret; +} + +static const struct file_operations qm_algqos_fops = { + .owner = THIS_MODULE, + .open = simple_open, + .read = qm_algqos_read, + .write = qm_algqos_write, +}; + +/** + * hisi_qm_set_algqos_init() - Initialize function qos debugfs files. + * @qm: The qm for which we want to add debugfs files. + * + * Create function qos debugfs files. + */ +static void hisi_qm_set_algqos_init(struct hisi_qm *qm) +{ + if (qm->fun_type == QM_HW_PF) + debugfs_create_file("alg_qos", 0644, qm->debug.debug_root, + qm, &qm_algqos_fops); + else + debugfs_create_file("alg_qos", 0444, qm->debug.debug_root, + qm, &qm_algqos_fops); +} + +/** + * hisi_qm_debug_init() - Initialize qm related debugfs files. + * @qm: The qm for which we want to add debugfs files. + * + * Create qm related debugfs files. + */ +void hisi_qm_debug_init(struct hisi_qm *qm) +{ + struct qm_dfx *dfx = &qm->debug.dfx; + struct dentry *qm_d; + void *data; + int i; + + qm_d = debugfs_create_dir("qm", qm->debug.debug_root); + qm->debug.qm_d = qm_d; + + /* only show this in PF */ + if (qm->fun_type == QM_HW_PF) { + qm_create_debugfs_file(qm, qm->debug.debug_root, CURRENT_QM); + for (i = CURRENT_Q; i < DEBUG_FILE_NUM; i++) + qm_create_debugfs_file(qm, qm->debug.qm_d, i); + } + + debugfs_create_file("regs", 0444, qm->debug.qm_d, qm, &qm_regs_fops); + + debugfs_create_file("cmd", 0600, qm->debug.qm_d, qm, &qm_cmd_fops); + + debugfs_create_file("status", 0444, qm->debug.qm_d, qm, + &qm_status_fops); + for (i = 0; i < ARRAY_SIZE(qm_dfx_files); i++) { + data = (atomic64_t *)((uintptr_t)dfx + qm_dfx_files[i].offset); + debugfs_create_file(qm_dfx_files[i].name, + 0644, + qm_d, + data, + &qm_atomic64_ops); + } + + if (qm->ver >= QM_HW_V3) + hisi_qm_set_algqos_init(qm); +} +EXPORT_SYMBOL_GPL(hisi_qm_debug_init); + +/** + * hisi_qm_debug_regs_clear() - clear qm debug related registers. + * @qm: The qm for which we want to clear its debug registers. + */ +void hisi_qm_debug_regs_clear(struct hisi_qm *qm) +{ + struct qm_dfx_registers *regs; + int i; + + /* clear current_qm */ + writel(0x0, qm->io_base + QM_DFX_MB_CNT_VF); + writel(0x0, qm->io_base + QM_DFX_DB_CNT_VF); + + /* clear current_q */ + writel(0x0, qm->io_base + QM_DFX_SQE_CNT_VF_SQN); + writel(0x0, qm->io_base + QM_DFX_CQE_CNT_VF_CQN); + + /* + * these registers are reading and clearing, so clear them after + * reading them. + */ + writel(0x1, qm->io_base + QM_DFX_CNT_CLR_CE); + + regs = qm_dfx_regs; + for (i = 0; i < CNT_CYC_REGS_NUM; i++) { + readl(qm->io_base + regs->reg_offset); + regs++; + } + + /* clear clear_enable */ + writel(0x0, qm->io_base + QM_DFX_CNT_CLR_CE); +} +EXPORT_SYMBOL_GPL(hisi_qm_debug_regs_clear); + /** * hisi_qm_sriov_enable() - enable virtual functions * @pdev: the PCIe device @@ -3416,6 +4327,7 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_enable); int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) { struct hisi_qm *qm = pci_get_drvdata(pdev); + int total_vfs = pci_sriov_get_totalvfs(qm->pdev); if (pci_vfs_assigned(pdev)) { pci_err(pdev, "Failed to disable VFs as VFs are assigned!\n"); @@ -3429,6 +4341,9 @@ int hisi_qm_sriov_disable(struct pci_dev *pdev, bool is_frozen) } pci_disable_sriov(pdev); + /* clear vf function shaper configure array */ + memset(qm->factor + 1, 0, sizeof(struct qm_shaper_factor) * total_vfs); + return qm_clear_vft_config(qm); } EXPORT_SYMBOL_GPL(hisi_qm_sriov_disable); @@ -3527,17 +4442,15 @@ pci_ers_result_t hisi_qm_dev_err_detected(struct pci_dev *pdev, } EXPORT_SYMBOL_GPL(hisi_qm_dev_err_detected); -static u32 qm_get_hw_error_status(struct hisi_qm *qm) -{ - return readl(qm->io_base + QM_ABNORMAL_INT_STATUS); -} - static int qm_check_req_recv(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; int ret; u32 val; + if (qm->ver >= QM_HW_V3) + return 0; + writel(ACC_VENDOR_ID_VALUE, qm->io_base + QM_PEH_VENDOR_ID); ret = readl_relaxed_poll_timeout(qm->io_base + QM_PEH_VENDOR_ID, val, (val == ACC_VENDOR_ID_VALUE), @@ -3608,28 +4521,6 @@ static int qm_set_vf_mse(struct hisi_qm *qm, bool set) return -ETIMEDOUT; } -static int qm_set_msi(struct hisi_qm *qm, bool set) -{ - struct pci_dev *pdev = qm->pdev; - - if (set) { - pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, - 0); - } else { - pci_write_config_dword(pdev, pdev->msi_cap + PCI_MSI_MASK_64, - ACC_PEH_MSI_DISABLE); - if (qm->err_status.is_qm_ecc_mbit || - qm->err_status.is_dev_ecc_mbit) - return 0; - - mdelay(1); - if (readl(qm->io_base + QM_PEH_DFX_INFO0)) - return -EFAULT; - } - - return 0; -} - static int qm_vf_reset_prepare(struct hisi_qm *qm, enum qm_stop_reason stop_reason) { @@ -3660,14 +4551,35 @@ stop_fail: return ret; } -static int qm_reset_prepare_ready(struct hisi_qm *qm) +static int qm_try_stop_vfs(struct hisi_qm *qm, u64 cmd, + enum qm_stop_reason stop_reason) { struct pci_dev *pdev = qm->pdev; - struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + int ret; + + if (!qm->vfs_num) + return 0; + + /* Kunpeng930 supports to notify VFs to stop before PF reset */ + if (qm->ops->ping_all_vfs) { + ret = qm->ops->ping_all_vfs(qm, cmd); + if (ret) + pci_err(pdev, "failed to send cmd to all VFs before PF reset!\n"); + } else { + ret = qm_vf_reset_prepare(qm, stop_reason); + if (ret) + pci_err(pdev, "failed to prepare reset, ret = %d.\n", ret); + } + + return ret; +} + +static int qm_wait_reset_finish(struct hisi_qm *qm) +{ int delay = 0; /* All reset requests need to be queued for processing */ - while (test_and_set_bit(QM_RESETTING, &pf_qm->misc_ctl)) { + while (test_and_set_bit(QM_RESETTING, &qm->misc_ctl)) { msleep(++delay); if (delay > QM_RESET_WAIT_TIMEOUT) return -EBUSY; @@ -3676,6 +4588,32 @@ static int qm_reset_prepare_ready(struct hisi_qm *qm) return 0; } +static int qm_reset_prepare_ready(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + /* + * PF and VF on host doesnot support resetting at the + * same time on Kunpeng920. + */ + if (qm->ver < QM_HW_V3) + return qm_wait_reset_finish(pf_qm); + + return qm_wait_reset_finish(qm); +} + +static void qm_reset_bit_clear(struct hisi_qm *qm) +{ + struct pci_dev *pdev = qm->pdev; + struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); + + if (qm->ver < QM_HW_V3) + clear_bit(QM_RESETTING, &pf_qm->misc_ctl); + + clear_bit(QM_RESETTING, &qm->misc_ctl); +} + static int qm_controller_reset_prepare(struct hisi_qm *qm) { struct pci_dev *pdev = qm->pdev; @@ -3687,22 +4625,25 @@ static int qm_controller_reset_prepare(struct hisi_qm *qm) return ret; } - if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm, QM_SOFT_RESET); - if (ret) { - pci_err(pdev, "Fails to stop VFs!\n"); - clear_bit(QM_RESETTING, &qm->misc_ctl); - return ret; - } - } + /* PF obtains the information of VF by querying the register. */ + qm_cmd_uninit(qm); + + /* Whether VFs stop successfully, soft reset will continue. */ + ret = qm_try_stop_vfs(qm, QM_PF_SRST_PREPARE, QM_SOFT_RESET); + if (ret) + pci_err(pdev, "failed to stop vfs by pf in soft reset.\n"); ret = hisi_qm_stop(qm, QM_SOFT_RESET); if (ret) { pci_err(pdev, "Fails to stop QM!\n"); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return ret; } + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to stop by vfs in soft reset!\n"); + clear_bit(QM_RST_SCHED, &qm->misc_ctl); return 0; @@ -3712,6 +4653,10 @@ static void qm_dev_ecc_mbit_handle(struct hisi_qm *qm) { u32 nfe_enb = 0; + /* Kunpeng930 hardware automatically close master ooo when NFE occurs */ + if (qm->ver >= QM_HW_V3) + return; + if (!qm->err_status.is_dev_ecc_mbit && qm->err_status.is_qm_ecc_mbit && qm->err_ini->close_axi_master_ooo) { @@ -3748,7 +4693,7 @@ static int qm_soft_reset(struct hisi_qm *qm) } } - ret = qm_set_msi(qm, false); + ret = qm->ops->set_msi(qm, false); if (ret) { pci_err(pdev, "Fails to disable PEH MSI bit.\n"); return ret; @@ -3770,6 +4715,9 @@ static int qm_soft_reset(struct hisi_qm *qm) return ret; } + if (qm->err_ini->close_sva_prefetch) + qm->err_ini->close_sva_prefetch(qm); + ret = qm_set_pf_mse(qm, false); if (ret) { pci_err(pdev, "Fails to disable pf MSE bit.\n"); @@ -3830,9 +4778,32 @@ restart_fail: return ret; } -static u32 qm_get_dev_err_status(struct hisi_qm *qm) +static int qm_try_start_vfs(struct hisi_qm *qm, enum qm_mb_cmd cmd) { - return qm->err_ini->get_dev_hw_err_status(qm); + struct pci_dev *pdev = qm->pdev; + int ret; + + if (!qm->vfs_num) + return 0; + + ret = qm_vf_q_assign(qm, qm->vfs_num); + if (ret) { + pci_err(pdev, "failed to assign VFs, ret = %d.\n", ret); + return ret; + } + + /* Kunpeng930 supports to notify VFs to start after PF reset. */ + if (qm->ops->ping_all_vfs) { + ret = qm->ops->ping_all_vfs(qm, cmd); + if (ret) + pci_warn(pdev, "failed to send cmd to all VFs after PF reset!\n"); + } else { + ret = qm_vf_reset_done(qm); + if (ret) + pci_warn(pdev, "failed to start vfs, ret = %d.\n", ret); + } + + return ret; } static int qm_dev_hw_init(struct hisi_qm *qm) @@ -3844,6 +4815,12 @@ static void qm_restart_prepare(struct hisi_qm *qm) { u32 value; + if (qm->err_ini->open_sva_prefetch) + qm->err_ini->open_sva_prefetch(qm); + + if (qm->ver >= QM_HW_V3) + return; + if (!qm->err_status.is_qm_ecc_mbit && !qm->err_status.is_dev_ecc_mbit) return; @@ -3863,15 +4840,15 @@ static void qm_restart_prepare(struct hisi_qm *qm) /* clear AM Reorder Buffer ecc mbit source */ writel(ACC_ROB_ECC_ERR_MULTPL, qm->io_base + ACC_AM_ROB_ECC_INT_STS); - - if (qm->err_ini->open_axi_master_ooo) - qm->err_ini->open_axi_master_ooo(qm); } static void qm_restart_done(struct hisi_qm *qm) { u32 value; + if (qm->ver >= QM_HW_V3) + goto clear_flags; + if (!qm->err_status.is_qm_ecc_mbit && !qm->err_status.is_dev_ecc_mbit) return; @@ -3881,6 +4858,7 @@ static void qm_restart_done(struct hisi_qm *qm) value |= qm->err_info.msi_wr_port; writel(value, qm->io_base + ACC_AM_CFG_PORT_WR_EN); +clear_flags: qm->err_status.is_qm_ecc_mbit = false; qm->err_status.is_dev_ecc_mbit = false; } @@ -3890,7 +4868,7 @@ static int qm_controller_reset_done(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; int ret; - ret = qm_set_msi(qm, true); + ret = qm->ops->set_msi(qm, true); if (ret) { pci_err(pdev, "Fails to enable PEH MSI bit!\n"); return ret; @@ -3917,6 +4895,9 @@ static int qm_controller_reset_done(struct hisi_qm *qm) } qm_restart_prepare(qm); + hisi_qm_dev_err_init(qm); + if (qm->err_ini->open_axi_master_ooo) + qm->err_ini->open_axi_master_ooo(qm); ret = qm_restart(qm); if (ret) { @@ -3924,24 +4905,18 @@ static int qm_controller_reset_done(struct hisi_qm *qm) return ret; } - if (qm->vfs_num) { - ret = qm_vf_q_assign(qm, qm->vfs_num); - if (ret) { - pci_err(pdev, "Failed to assign queue!\n"); - return ret; - } - } + ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); + if (ret) + pci_err(pdev, "failed to start vfs by pf in soft reset.\n"); - ret = qm_vf_reset_done(qm); - if (ret) { - pci_err(pdev, "Failed to start VFs!\n"); - return -EPERM; - } + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to start by vfs in soft reset!\n"); - hisi_qm_dev_err_init(qm); + qm_cmd_init(qm); qm_restart_done(qm); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return 0; } @@ -3962,13 +4937,13 @@ static int qm_controller_reset(struct hisi_qm *qm) ret = qm_soft_reset(qm); if (ret) { pci_err(pdev, "Controller reset failed (%d)\n", ret); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return ret; } ret = qm_controller_reset_done(qm); if (ret) { - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); return ret; } @@ -4005,21 +4980,6 @@ pci_ers_result_t hisi_qm_dev_slot_reset(struct pci_dev *pdev) } EXPORT_SYMBOL_GPL(hisi_qm_dev_slot_reset); -/* check the interrupt is ecc-mbit error or not */ -static int qm_check_dev_error(struct hisi_qm *qm) -{ - int ret; - - if (qm->fun_type == QM_HW_VF) - return 0; - - ret = qm_get_hw_error_status(qm) & QM_ECC_MBIT; - if (ret) - return ret; - - return (qm_get_dev_err_status(qm) & qm->err_info.ecc_2bits_mask); -} - void hisi_qm_reset_prepare(struct pci_dev *pdev) { struct hisi_qm *pf_qm = pci_get_drvdata(pci_physfn(pdev)); @@ -4045,14 +5005,13 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) return; } - if (qm->vfs_num) { - ret = qm_vf_reset_prepare(qm, QM_FLR); - if (ret) { - pci_err(pdev, "Failed to prepare reset, ret = %d.\n", - ret); - return; - } - } + /* PF obtains the information of VF by querying the register. */ + if (qm->fun_type == QM_HW_PF) + qm_cmd_uninit(qm); + + ret = qm_try_stop_vfs(qm, QM_PF_FLR_PREPARE, QM_FLR); + if (ret) + pci_err(pdev, "failed to stop vfs by pf in FLR.\n"); ret = hisi_qm_stop(qm, QM_FLR); if (ret) { @@ -4060,6 +5019,10 @@ void hisi_qm_reset_prepare(struct pci_dev *pdev) return; } + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to stop by vfs in FLR!\n"); + pci_info(pdev, "FLR resetting...\n"); } EXPORT_SYMBOL_GPL(hisi_qm_reset_prepare); @@ -4085,42 +5048,38 @@ void hisi_qm_reset_done(struct pci_dev *pdev) struct hisi_qm *qm = pci_get_drvdata(pdev); int ret; - hisi_qm_dev_err_init(pf_qm); - - ret = qm_restart(qm); - if (ret) { - pci_err(pdev, "Failed to start QM, ret = %d.\n", ret); - goto flr_done; - } - if (qm->fun_type == QM_HW_PF) { ret = qm_dev_hw_init(qm); if (ret) { pci_err(pdev, "Failed to init PF, ret = %d.\n", ret); goto flr_done; } + } - if (!qm->vfs_num) - goto flr_done; - - ret = qm_vf_q_assign(qm, qm->vfs_num); - if (ret) { - pci_err(pdev, "Failed to assign VFs, ret = %d.\n", ret); - goto flr_done; - } + hisi_qm_dev_err_init(pf_qm); - ret = qm_vf_reset_done(qm); - if (ret) { - pci_err(pdev, "Failed to start VFs, ret = %d.\n", ret); - goto flr_done; - } + ret = qm_restart(qm); + if (ret) { + pci_err(pdev, "Failed to start QM, ret = %d.\n", ret); + goto flr_done; } + ret = qm_try_start_vfs(qm, QM_PF_RESET_DONE); + if (ret) + pci_err(pdev, "failed to start vfs by pf in FLR.\n"); + + ret = qm_wait_vf_prepare_finish(qm); + if (ret) + pci_err(pdev, "failed to start by vfs in FLR!\n"); + flr_done: + if (qm->fun_type == QM_HW_PF) + qm_cmd_init(qm); + if (qm_flr_reset_complete(pdev)) pci_info(pdev, "FLR reset complete\n"); - clear_bit(QM_RESETTING, &qm->misc_ctl); + qm_reset_bit_clear(qm); } EXPORT_SYMBOL_GPL(hisi_qm_reset_done); @@ -4149,7 +5108,7 @@ static int qm_irq_register(struct hisi_qm *qm) if (ret) return ret; - if (qm->ver != QM_HW_V1) { + if (qm->ver > QM_HW_V1) { ret = request_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm_aeq_irq, 0, qm->dev_name, qm); if (ret) @@ -4164,8 +5123,18 @@ static int qm_irq_register(struct hisi_qm *qm) } } + if (qm->ver > QM_HW_V2) { + ret = request_irq(pci_irq_vector(pdev, QM_CMD_EVENT_IRQ_VECTOR), + qm_mb_cmd_irq, 0, qm->dev_name, qm); + if (ret) + goto err_mb_cmd_irq; + } + return 0; +err_mb_cmd_irq: + if (qm->fun_type == QM_HW_PF) + free_irq(pci_irq_vector(pdev, QM_ABNORMAL_EVENT_IRQ_VECTOR), qm); err_abonormal_irq: free_irq(pci_irq_vector(pdev, QM_AEQ_EVENT_IRQ_VECTOR), qm); err_aeq_irq: @@ -4202,6 +5171,183 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) } +static void qm_pf_reset_vf_prepare(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) +{ + enum qm_mb_cmd cmd = QM_VF_PREPARE_DONE; + struct pci_dev *pdev = qm->pdev; + int ret; + + ret = qm_reset_prepare_ready(qm); + if (ret) { + dev_err(&pdev->dev, "reset prepare not ready!\n"); + atomic_set(&qm->status.flags, QM_STOP); + cmd = QM_VF_PREPARE_FAIL; + goto err_prepare; + } + + ret = hisi_qm_stop(qm, stop_reason); + if (ret) { + dev_err(&pdev->dev, "failed to stop QM, ret = %d.\n", ret); + atomic_set(&qm->status.flags, QM_STOP); + cmd = QM_VF_PREPARE_FAIL; + goto err_prepare; + } + +err_prepare: + pci_save_state(pdev); + ret = qm->ops->ping_pf(qm, cmd); + if (ret) + dev_warn(&pdev->dev, "PF responds timeout in reset prepare!\n"); +} + +static void qm_pf_reset_vf_done(struct hisi_qm *qm) +{ + enum qm_mb_cmd cmd = QM_VF_START_DONE; + struct pci_dev *pdev = qm->pdev; + int ret; + + pci_restore_state(pdev); + ret = hisi_qm_start(qm); + if (ret) { + dev_err(&pdev->dev, "failed to start QM, ret = %d.\n", ret); + cmd = QM_VF_START_FAIL; + } + + ret = qm->ops->ping_pf(qm, cmd); + if (ret) + dev_warn(&pdev->dev, "PF responds timeout in reset done!\n"); + + qm_reset_bit_clear(qm); +} + +static int qm_wait_pf_reset_finish(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + u32 val, cmd; + u64 msg; + int ret; + + /* Wait for reset to finish */ + ret = readl_relaxed_poll_timeout(qm->io_base + QM_IFC_INT_SOURCE_V, val, + val == BIT(0), QM_VF_RESET_WAIT_US, + QM_VF_RESET_WAIT_TIMEOUT_US); + /* hardware completion status should be available by this time */ + if (ret) { + dev_err(dev, "couldn't get reset done status from PF, timeout!\n"); + return -ETIMEDOUT; + } + + /* + * Whether message is got successfully, + * VF needs to ack PF by clearing the interrupt. + */ + ret = qm_get_mb_cmd(qm, &msg, 0); + qm_clear_cmd_interrupt(qm, 0); + if (ret) { + dev_err(dev, "failed to get msg from PF in reset done!\n"); + return ret; + } + + cmd = msg & QM_MB_CMD_DATA_MASK; + if (cmd != QM_PF_RESET_DONE) { + dev_err(dev, "the cmd(%u) is not reset done!\n", cmd); + ret = -EINVAL; + } + + return ret; +} + +static void qm_pf_reset_vf_process(struct hisi_qm *qm, + enum qm_stop_reason stop_reason) +{ + struct device *dev = &qm->pdev->dev; + int ret; + + dev_info(dev, "device reset start...\n"); + + /* The message is obtained by querying the register during resetting */ + qm_cmd_uninit(qm); + qm_pf_reset_vf_prepare(qm, stop_reason); + + ret = qm_wait_pf_reset_finish(qm); + if (ret) + goto err_get_status; + + qm_pf_reset_vf_done(qm); + qm_cmd_init(qm); + + dev_info(dev, "device reset done.\n"); + + return; + +err_get_status: + qm_cmd_init(qm); + qm_reset_bit_clear(qm); +} + +static void qm_handle_cmd_msg(struct hisi_qm *qm, u32 fun_num) +{ + struct device *dev = &qm->pdev->dev; + u64 msg; + u32 cmd; + int ret; + + /* + * Get the msg from source by sending mailbox. Whether message is got + * successfully, destination needs to ack source by clearing the interrupt. + */ + ret = qm_get_mb_cmd(qm, &msg, fun_num); + qm_clear_cmd_interrupt(qm, BIT(fun_num)); + if (ret) { + dev_err(dev, "failed to get msg from source!\n"); + return; + } + + cmd = msg & QM_MB_CMD_DATA_MASK; + switch (cmd) { + case QM_PF_FLR_PREPARE: + qm_pf_reset_vf_process(qm, QM_FLR); + break; + case QM_PF_SRST_PREPARE: + qm_pf_reset_vf_process(qm, QM_SOFT_RESET); + break; + case QM_VF_GET_QOS: + qm_vf_get_qos(qm, fun_num); + break; + case QM_PF_SET_QOS: + qm->mb_qos = msg >> QM_MB_CMD_DATA_SHIFT; + break; + default: + dev_err(dev, "unsupported cmd %u sent by function(%u)!\n", cmd, fun_num); + break; + } +} + +static void qm_cmd_process(struct work_struct *cmd_process) +{ + struct hisi_qm *qm = container_of(cmd_process, + struct hisi_qm, cmd_process); + u32 vfs_num = qm->vfs_num; + u64 val; + u32 i; + + if (qm->fun_type == QM_HW_PF) { + val = readq(qm->io_base + QM_IFC_INT_SOURCE_P); + if (!val) + return; + + for (i = 1; i <= vfs_num; i++) { + if (val & BIT(i)) + qm_handle_cmd_msg(qm, i); + } + + return; + } + + qm_handle_cmd_msg(qm, 0); +} + /** * hisi_qm_alg_register() - Register alg to crypto and add qm to qm_list. * @qm: The qm needs add. @@ -4212,11 +5358,9 @@ static void hisi_qm_controller_reset(struct work_struct *rst_work) */ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { + struct device *dev = &qm->pdev->dev; int flag = 0; int ret = 0; - /* HW V2 not support both use uacce sva mode and hardware crypto algs */ - if (qm->ver <= QM_HW_V2 && qm->use_sva) - return 0; mutex_lock(&qm_list->lock); if (list_empty(&qm_list->list)) @@ -4224,6 +5368,11 @@ int hisi_qm_alg_register(struct hisi_qm *qm, struct hisi_qm_list *qm_list) list_add_tail(&qm->list, &qm_list->list); mutex_unlock(&qm_list->lock); + if (qm->ver <= QM_HW_V2 && qm->use_sva) { + dev_info(dev, "HW V2 not both use uacce sva mode and hardware crypto algs.\n"); + return 0; + } + if (flag) { ret = qm_list->register_to_crypto(qm); if (ret) { @@ -4248,13 +5397,13 @@ EXPORT_SYMBOL_GPL(hisi_qm_alg_register); */ void hisi_qm_alg_unregister(struct hisi_qm *qm, struct hisi_qm_list *qm_list) { - if (qm->ver <= QM_HW_V2 && qm->use_sva) - return; - mutex_lock(&qm_list->lock); list_del(&qm->list); mutex_unlock(&qm_list->lock); + if (qm->ver <= QM_HW_V2 && qm->use_sva) + return; + if (list_empty(&qm_list->list)) qm_list->unregister_from_crypto(qm); } @@ -4389,6 +5538,94 @@ err_disable_pcidev: return ret; } +static void hisi_qm_init_work(struct hisi_qm *qm) +{ + INIT_WORK(&qm->work, qm_work_process); + if (qm->fun_type == QM_HW_PF) + INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); + + if (qm->ver > QM_HW_V2) + INIT_WORK(&qm->cmd_process, qm_cmd_process); +} + +static int hisi_qp_alloc_memory(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + size_t qp_dma_size; + int i, ret; + + qm->qp_array = kcalloc(qm->qp_num, sizeof(struct hisi_qp), GFP_KERNEL); + if (!qm->qp_array) + return -ENOMEM; + + /* one more page for device or qp statuses */ + qp_dma_size = qm->sqe_size * QM_Q_DEPTH + + sizeof(struct qm_cqe) * QM_Q_DEPTH; + qp_dma_size = PAGE_ALIGN(qp_dma_size) + PAGE_SIZE; + for (i = 0; i < qm->qp_num; i++) { + ret = hisi_qp_memory_init(qm, qp_dma_size, i); + if (ret) + goto err_init_qp_mem; + + dev_dbg(dev, "allocate qp dma buf size=%zx)\n", qp_dma_size); + } + + return 0; +err_init_qp_mem: + hisi_qp_memory_uninit(qm, i); + + return ret; +} + +static int hisi_qm_memory_init(struct hisi_qm *qm) +{ + struct device *dev = &qm->pdev->dev; + int ret, total_vfs; + size_t off = 0; + + total_vfs = pci_sriov_get_totalvfs(qm->pdev); + qm->factor = kcalloc(total_vfs + 1, sizeof(struct qm_shaper_factor), GFP_KERNEL); + if (!qm->factor) + return -ENOMEM; + +#define QM_INIT_BUF(qm, type, num) do { \ + (qm)->type = ((qm)->qdma.va + (off)); \ + (qm)->type##_dma = (qm)->qdma.dma + (off); \ + off += QMC_ALIGN(sizeof(struct qm_##type) * (num)); \ +} while (0) + + idr_init(&qm->qp_idr); + qm->qdma.size = QMC_ALIGN(sizeof(struct qm_eqe) * QM_EQ_DEPTH) + + QMC_ALIGN(sizeof(struct qm_aeqe) * QM_Q_DEPTH) + + QMC_ALIGN(sizeof(struct qm_sqc) * qm->qp_num) + + QMC_ALIGN(sizeof(struct qm_cqc) * qm->qp_num); + qm->qdma.va = dma_alloc_coherent(dev, qm->qdma.size, &qm->qdma.dma, + GFP_ATOMIC); + dev_dbg(dev, "allocate qm dma buf size=%zx)\n", qm->qdma.size); + if (!qm->qdma.va) { + ret = -ENOMEM; + goto err_alloc_qdma; + } + + QM_INIT_BUF(qm, eqe, QM_EQ_DEPTH); + QM_INIT_BUF(qm, aeqe, QM_Q_DEPTH); + QM_INIT_BUF(qm, sqc, qm->qp_num); + QM_INIT_BUF(qm, cqc, qm->qp_num); + + ret = hisi_qp_alloc_memory(qm); + if (ret) + goto err_alloc_qp_array; + + return 0; + +err_alloc_qp_array: + dma_free_coherent(dev, qm->qdma.size, qm->qdma.va, qm->qdma.dma); +err_alloc_qdma: + kfree(qm->factor); + + return ret; +} + /** * hisi_qm_init() - Initialize configures about qm. * @qm: The qm needing init. @@ -4426,10 +5663,8 @@ int hisi_qm_init(struct hisi_qm *qm) if (ret) goto err_alloc_uacce; - INIT_WORK(&qm->work, qm_work_process); - if (qm->fun_type == QM_HW_PF) - INIT_WORK(&qm->rst_work, hisi_qm_controller_reset); - + hisi_qm_init_work(qm); + qm_cmd_init(qm); atomic_set(&qm->status.flags, QM_INIT); return 0; diff --git a/drivers/crypto/hisilicon/qm.h b/drivers/crypto/hisilicon/qm.h index acefdf8b3a50..035eaf8c442d 100644 --- a/drivers/crypto/hisilicon/qm.h +++ b/drivers/crypto/hisilicon/qm.h @@ -76,6 +76,9 @@ #define QM_Q_DEPTH 1024 #define QM_MIN_QNUM 2 #define HISI_ACC_SGL_SGE_NR_MAX 255 +#define QM_SHAPER_CFG 0x100164 +#define QM_SHAPER_ENABLE BIT(30) +#define QM_SHAPER_TYPE1_OFFSET 10 /* page number for queue file region */ #define QM_DOORBELL_PAGE_NR 1 @@ -148,6 +151,14 @@ struct qm_debug { struct debugfs_file files[DEBUG_FILE_NUM]; }; +struct qm_shaper_factor { + u32 func_qos; + u64 cir_b; + u64 cir_u; + u64 cir_s; + u64 cbs_s; +}; + struct qm_dma { void *va; dma_addr_t dma; @@ -188,6 +199,8 @@ struct hisi_qm_err_ini { void (*clear_dev_hw_err_status)(struct hisi_qm *qm, u32 err_sts); void (*open_axi_master_ooo)(struct hisi_qm *qm); void (*close_axi_master_ooo)(struct hisi_qm *qm); + void (*open_sva_prefetch)(struct hisi_qm *qm); + void (*close_sva_prefetch)(struct hisi_qm *qm); void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts); void (*err_info_init)(struct hisi_qm *qm); }; @@ -248,6 +261,7 @@ struct hisi_qm { struct workqueue_struct *wq; struct work_struct work; struct work_struct rst_work; + struct work_struct cmd_process; const char *algs; bool use_sva; @@ -259,6 +273,9 @@ struct hisi_qm { resource_size_t db_phys_base; struct uacce_device *uacce; int mode; + struct qm_shaper_factor *factor; + u32 mb_qos; + u32 type_rate; }; struct hisi_qp_status { diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index dfdce2f21e65..018415b9840a 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -13,14 +13,14 @@ struct sec_alg_res { dma_addr_t pbuf_dma; u8 *c_ivin; dma_addr_t c_ivin_dma; + u8 *a_ivin; + dma_addr_t a_ivin_dma; u8 *out_mac; dma_addr_t out_mac_dma; }; /* Cipher request of SEC private */ struct sec_cipher_req { - struct hisi_acc_hw_sgl *c_in; - dma_addr_t c_in_dma; struct hisi_acc_hw_sgl *c_out; dma_addr_t c_out_dma; u8 *c_ivin; @@ -33,15 +33,25 @@ struct sec_cipher_req { struct sec_aead_req { u8 *out_mac; dma_addr_t out_mac_dma; + u8 *a_ivin; + dma_addr_t a_ivin_dma; struct aead_request *aead_req; }; /* SEC request of Crypto */ struct sec_req { - struct sec_sqe sec_sqe; + union { + struct sec_sqe sec_sqe; + struct sec_sqe3 sec_sqe3; + }; struct sec_ctx *ctx; struct sec_qp_ctx *qp_ctx; + /** + * Common parameter of the SEC request. + */ + struct hisi_acc_hw_sgl *in; + dma_addr_t in_dma; struct sec_cipher_req c_req; struct sec_aead_req aead_req; struct list_head backlog_head; @@ -81,7 +91,9 @@ struct sec_auth_ctx { u8 a_key_len; u8 mac_len; u8 a_alg; + bool fallback; struct crypto_shash *hash_tfm; + struct crypto_aead *fallback_aead_tfm; }; /* SEC cipher context which cipher's relatives */ @@ -94,6 +106,10 @@ struct sec_cipher_ctx { u8 c_mode; u8 c_alg; u8 c_key_len; + + /* add software support */ + bool fallback; + struct crypto_sync_skcipher *fbtfm; }; /* SEC queue context which defines queue's relatives */ @@ -137,6 +153,7 @@ struct sec_ctx { bool pbuf_supported; struct sec_cipher_ctx c_ctx; struct sec_auth_ctx a_ctx; + u8 type_supported; struct device *dev; }; diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 133aede8bf07..6a45bd23b363 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -2,6 +2,7 @@ /* Copyright (c) 2019 HiSilicon Limited. */ #include <crypto/aes.h> +#include <crypto/aead.h> #include <crypto/algapi.h> #include <crypto/authenc.h> #include <crypto/des.h> @@ -21,6 +22,7 @@ #define SEC_PRIORITY 4001 #define SEC_XTS_MIN_KEY_SIZE (2 * AES_MIN_KEY_SIZE) +#define SEC_XTS_MID_KEY_SIZE (3 * AES_MIN_KEY_SIZE) #define SEC_XTS_MAX_KEY_SIZE (2 * AES_MAX_KEY_SIZE) #define SEC_DES3_2KEY_SIZE (2 * DES_KEY_SIZE) #define SEC_DES3_3KEY_SIZE (3 * DES_KEY_SIZE) @@ -37,10 +39,23 @@ #define SEC_AEAD_ALG_OFFSET 11 #define SEC_AUTH_OFFSET 6 +#define SEC_DE_OFFSET_V3 9 +#define SEC_SCENE_OFFSET_V3 5 +#define SEC_CKEY_OFFSET_V3 13 +#define SEC_SRC_SGL_OFFSET_V3 11 +#define SEC_DST_SGL_OFFSET_V3 14 +#define SEC_CALG_OFFSET_V3 4 +#define SEC_AKEY_OFFSET_V3 9 +#define SEC_MAC_OFFSET_V3 4 +#define SEC_AUTH_ALG_OFFSET_V3 15 +#define SEC_CIPHER_AUTH_V3 0xbf +#define SEC_AUTH_CIPHER_V3 0x40 #define SEC_FLAG_OFFSET 7 #define SEC_FLAG_MASK 0x0780 #define SEC_TYPE_MASK 0x0F #define SEC_DONE_MASK 0x0001 +#define SEC_ICV_MASK 0x000E +#define SEC_SQE_LEN_RATE_MASK 0x3 #define SEC_TOTAL_IV_SZ (SEC_IV_SIZE * QM_Q_DEPTH) #define SEC_SGL_SGE_NR 128 @@ -66,6 +81,25 @@ #define SEC_SQE_CFLAG 2 #define SEC_SQE_AEAD_FLAG 3 #define SEC_SQE_DONE 0x1 +#define SEC_ICV_ERR 0x2 +#define MIN_MAC_LEN 4 +#define MAC_LEN_MASK 0x1U +#define MAX_INPUT_DATA_LEN 0xFFFE00 +#define BITS_MASK 0xFF +#define BYTE_BITS 0x8 +#define SEC_XTS_NAME_SZ 0x3 +#define IV_CM_CAL_NUM 2 +#define IV_CL_MASK 0x7 +#define IV_CL_MIN 2 +#define IV_CL_MID 4 +#define IV_CL_MAX 8 +#define IV_FLAGS_OFFSET 0x6 +#define IV_CM_OFFSET 0x3 +#define IV_LAST_BYTE1 1 +#define IV_LAST_BYTE2 2 +#define IV_LAST_BYTE_MASK 0xFF +#define IV_CTR_INIT 0x1 +#define IV_BYTE_OFFSET 0x8 /* Get an en/de-cipher queue cyclically to balance load over queues of TFM */ static inline int sec_alloc_queue_id(struct sec_ctx *ctx, struct sec_req *req) @@ -124,22 +158,59 @@ static void sec_free_req_id(struct sec_req *req) mutex_unlock(&qp_ctx->req_lock); } -static int sec_aead_verify(struct sec_req *req) +static u8 pre_parse_finished_bd(struct bd_status *status, void *resp) { - struct aead_request *aead_req = req->aead_req.aead_req; - struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); - size_t authsize = crypto_aead_authsize(tfm); - u8 *mac_out = req->aead_req.out_mac; - u8 *mac = mac_out + SEC_MAX_MAC_LEN; - struct scatterlist *sgl = aead_req->src; - size_t sz; + struct sec_sqe *bd = resp; + + status->done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; + status->icv = (le16_to_cpu(bd->type2.done_flag) & SEC_ICV_MASK) >> 1; + status->flag = (le16_to_cpu(bd->type2.done_flag) & + SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; + status->tag = le16_to_cpu(bd->type2.tag); + status->err_type = bd->type2.error_type; + + return bd->type_cipher_auth & SEC_TYPE_MASK; +} + +static u8 pre_parse_finished_bd3(struct bd_status *status, void *resp) +{ + struct sec_sqe3 *bd3 = resp; + + status->done = le16_to_cpu(bd3->done_flag) & SEC_DONE_MASK; + status->icv = (le16_to_cpu(bd3->done_flag) & SEC_ICV_MASK) >> 1; + status->flag = (le16_to_cpu(bd3->done_flag) & + SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; + status->tag = le64_to_cpu(bd3->tag); + status->err_type = bd3->error_type; + + return le32_to_cpu(bd3->bd_param) & SEC_TYPE_MASK; +} + +static int sec_cb_status_check(struct sec_req *req, + struct bd_status *status) +{ + struct sec_ctx *ctx = req->ctx; - sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac, authsize, - aead_req->cryptlen + aead_req->assoclen - - authsize); - if (unlikely(sz != authsize || memcmp(mac_out, mac, sz))) { - dev_err(req->ctx->dev, "aead verify failure!\n"); - return -EBADMSG; + if (unlikely(req->err_type || status->done != SEC_SQE_DONE)) { + dev_err_ratelimited(ctx->dev, "err_type[%d], done[%u]\n", + req->err_type, status->done); + return -EIO; + } + + if (unlikely(ctx->alg_type == SEC_SKCIPHER)) { + if (unlikely(status->flag != SEC_SQE_CFLAG)) { + dev_err_ratelimited(ctx->dev, "flag[%u]\n", + status->flag); + return -EIO; + } + } else if (unlikely(ctx->alg_type == SEC_AEAD)) { + if (unlikely(status->flag != SEC_SQE_AEAD_FLAG || + status->icv == SEC_ICV_ERR)) { + dev_err_ratelimited(ctx->dev, + "flag[%u], icv[%u]\n", + status->flag, status->icv); + return -EBADMSG; + } } return 0; @@ -149,43 +220,38 @@ static void sec_req_cb(struct hisi_qp *qp, void *resp) { struct sec_qp_ctx *qp_ctx = qp->qp_ctx; struct sec_dfx *dfx = &qp_ctx->ctx->sec->debug.dfx; - struct sec_sqe *bd = resp; + u8 type_supported = qp_ctx->ctx->type_supported; + struct bd_status status; struct sec_ctx *ctx; struct sec_req *req; - u16 done, flag; - int err = 0; + int err; u8 type; - type = bd->type_cipher_auth & SEC_TYPE_MASK; - if (unlikely(type != SEC_BD_TYPE2)) { + if (type_supported == SEC_BD_TYPE2) { + type = pre_parse_finished_bd(&status, resp); + req = qp_ctx->req_list[status.tag]; + } else { + type = pre_parse_finished_bd3(&status, resp); + req = (void *)(uintptr_t)status.tag; + } + + if (unlikely(type != type_supported)) { atomic64_inc(&dfx->err_bd_cnt); pr_err("err bd type [%d]\n", type); return; } - req = qp_ctx->req_list[le16_to_cpu(bd->type2.tag)]; if (unlikely(!req)) { atomic64_inc(&dfx->invalid_req_cnt); atomic_inc(&qp->qp_status.used); return; } - req->err_type = bd->type2.error_type; + + req->err_type = status.err_type; ctx = req->ctx; - done = le16_to_cpu(bd->type2.done_flag) & SEC_DONE_MASK; - flag = (le16_to_cpu(bd->type2.done_flag) & - SEC_FLAG_MASK) >> SEC_FLAG_OFFSET; - if (unlikely(req->err_type || done != SEC_SQE_DONE || - (ctx->alg_type == SEC_SKCIPHER && flag != SEC_SQE_CFLAG) || - (ctx->alg_type == SEC_AEAD && flag != SEC_SQE_AEAD_FLAG))) { - dev_err_ratelimited(ctx->dev, - "err_type[%d],done[%d],flag[%d]\n", - req->err_type, done, flag); - err = -EIO; + err = sec_cb_status_check(req, &status); + if (err) atomic64_inc(&dfx->done_flag_cnt); - } - - if (ctx->alg_type == SEC_AEAD && !req->c_req.encrypt) - err = sec_aead_verify(req); atomic64_inc(&dfx->recv_cnt); @@ -253,6 +319,30 @@ static void sec_free_civ_resource(struct device *dev, struct sec_alg_res *res) res->c_ivin, res->c_ivin_dma); } +static int sec_alloc_aiv_resource(struct device *dev, struct sec_alg_res *res) +{ + int i; + + res->a_ivin = dma_alloc_coherent(dev, SEC_TOTAL_IV_SZ, + &res->a_ivin_dma, GFP_KERNEL); + if (!res->a_ivin) + return -ENOMEM; + + for (i = 1; i < QM_Q_DEPTH; i++) { + res[i].a_ivin_dma = res->a_ivin_dma + i * SEC_IV_SIZE; + res[i].a_ivin = res->a_ivin + i * SEC_IV_SIZE; + } + + return 0; +} + +static void sec_free_aiv_resource(struct device *dev, struct sec_alg_res *res) +{ + if (res->a_ivin) + dma_free_coherent(dev, SEC_TOTAL_IV_SZ, + res->a_ivin, res->a_ivin_dma); +} + static int sec_alloc_mac_resource(struct device *dev, struct sec_alg_res *res) { int i; @@ -335,9 +425,13 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, return ret; if (ctx->alg_type == SEC_AEAD) { + ret = sec_alloc_aiv_resource(dev, res); + if (ret) + goto alloc_aiv_fail; + ret = sec_alloc_mac_resource(dev, res); if (ret) - goto alloc_fail; + goto alloc_mac_fail; } if (ctx->pbuf_supported) { ret = sec_alloc_pbuf_resource(dev, res); @@ -352,7 +446,10 @@ static int sec_alg_resource_alloc(struct sec_ctx *ctx, alloc_pbuf_fail: if (ctx->alg_type == SEC_AEAD) sec_free_mac_resource(dev, qp_ctx->res); -alloc_fail: +alloc_mac_fail: + if (ctx->alg_type == SEC_AEAD) + sec_free_aiv_resource(dev, res); +alloc_aiv_fail: sec_free_civ_resource(dev, res); return ret; } @@ -382,10 +479,11 @@ static int sec_create_qp_ctx(struct hisi_qm *qm, struct sec_ctx *ctx, qp = ctx->qps[qp_ctx_id]; qp->req_type = 0; qp->qp_ctx = qp_ctx; - qp->req_cb = sec_req_cb; qp_ctx->qp = qp; qp_ctx->ctx = ctx; + qp->req_cb = sec_req_cb; + mutex_init(&qp_ctx->req_lock); idr_init(&qp_ctx->req_idr); INIT_LIST_HEAD(&qp_ctx->backlog); @@ -536,6 +634,26 @@ static void sec_auth_uninit(struct sec_ctx *ctx) a_ctx->a_key, a_ctx->a_key_dma); } +static int sec_skcipher_fbtfm_init(struct crypto_skcipher *tfm) +{ + const char *alg = crypto_tfm_alg_name(&tfm->base); + struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + + c_ctx->fallback = false; + if (likely(strncmp(alg, "xts", SEC_XTS_NAME_SZ))) + return 0; + + c_ctx->fbtfm = crypto_alloc_sync_skcipher(alg, 0, + CRYPTO_ALG_NEED_FALLBACK); + if (IS_ERR(c_ctx->fbtfm)) { + pr_err("failed to alloc fallback tfm!\n"); + return PTR_ERR(c_ctx->fbtfm); + } + + return 0; +} + static int sec_skcipher_init(struct crypto_skcipher *tfm) { struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); @@ -557,8 +675,14 @@ static int sec_skcipher_init(struct crypto_skcipher *tfm) if (ret) goto err_cipher_init; + ret = sec_skcipher_fbtfm_init(tfm); + if (ret) + goto err_fbtfm_init; + return 0; +err_fbtfm_init: + sec_cipher_uninit(ctx); err_cipher_init: sec_ctx_base_uninit(ctx); return ret; @@ -568,6 +692,9 @@ static void sec_skcipher_uninit(struct crypto_skcipher *tfm) { struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + if (ctx->c_ctx.fbtfm) + crypto_free_sync_skcipher(ctx->c_ctx.fbtfm); + sec_cipher_uninit(ctx); sec_ctx_base_uninit(ctx); } @@ -607,6 +734,9 @@ static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, case SEC_XTS_MIN_KEY_SIZE: c_ctx->c_key_len = SEC_CKEY_128BIT; break; + case SEC_XTS_MID_KEY_SIZE: + c_ctx->fallback = true; + break; case SEC_XTS_MAX_KEY_SIZE: c_ctx->c_key_len = SEC_CKEY_256BIT; break; @@ -615,19 +745,25 @@ static int sec_skcipher_aes_sm4_setkey(struct sec_cipher_ctx *c_ctx, return -EINVAL; } } else { - switch (keylen) { - case AES_KEYSIZE_128: - c_ctx->c_key_len = SEC_CKEY_128BIT; - break; - case AES_KEYSIZE_192: - c_ctx->c_key_len = SEC_CKEY_192BIT; - break; - case AES_KEYSIZE_256: - c_ctx->c_key_len = SEC_CKEY_256BIT; - break; - default: - pr_err("hisi_sec2: aes key error!\n"); + if (c_ctx->c_alg == SEC_CALG_SM4 && + keylen != AES_KEYSIZE_128) { + pr_err("hisi_sec2: sm4 key error!\n"); return -EINVAL; + } else { + switch (keylen) { + case AES_KEYSIZE_128: + c_ctx->c_key_len = SEC_CKEY_128BIT; + break; + case AES_KEYSIZE_192: + c_ctx->c_key_len = SEC_CKEY_192BIT; + break; + case AES_KEYSIZE_256: + c_ctx->c_key_len = SEC_CKEY_256BIT; + break; + default: + pr_err("hisi_sec2: aes key error!\n"); + return -EINVAL; + } } } @@ -672,7 +808,13 @@ static int sec_skcipher_setkey(struct crypto_skcipher *tfm, const u8 *key, } memcpy(c_ctx->c_key, key, keylen); - + if (c_ctx->fallback) { + ret = crypto_sync_skcipher_setkey(c_ctx->fbtfm, key, keylen); + if (ret) { + dev_err(dev, "failed to set fallback skcipher key!\n"); + return ret; + } + } return 0; } @@ -686,22 +828,30 @@ static int sec_setkey_##name(struct crypto_skcipher *tfm, const u8 *key,\ GEN_SEC_SETKEY_FUNC(aes_ecb, SEC_CALG_AES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(aes_cbc, SEC_CALG_AES, SEC_CMODE_CBC) GEN_SEC_SETKEY_FUNC(aes_xts, SEC_CALG_AES, SEC_CMODE_XTS) - +GEN_SEC_SETKEY_FUNC(aes_ofb, SEC_CALG_AES, SEC_CMODE_OFB) +GEN_SEC_SETKEY_FUNC(aes_cfb, SEC_CALG_AES, SEC_CMODE_CFB) +GEN_SEC_SETKEY_FUNC(aes_ctr, SEC_CALG_AES, SEC_CMODE_CTR) GEN_SEC_SETKEY_FUNC(3des_ecb, SEC_CALG_3DES, SEC_CMODE_ECB) GEN_SEC_SETKEY_FUNC(3des_cbc, SEC_CALG_3DES, SEC_CMODE_CBC) - GEN_SEC_SETKEY_FUNC(sm4_xts, SEC_CALG_SM4, SEC_CMODE_XTS) GEN_SEC_SETKEY_FUNC(sm4_cbc, SEC_CALG_SM4, SEC_CMODE_CBC) +GEN_SEC_SETKEY_FUNC(sm4_ofb, SEC_CALG_SM4, SEC_CMODE_OFB) +GEN_SEC_SETKEY_FUNC(sm4_cfb, SEC_CALG_SM4, SEC_CMODE_CFB) +GEN_SEC_SETKEY_FUNC(sm4_ctr, SEC_CALG_SM4, SEC_CMODE_CTR) static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, struct scatterlist *src) { - struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aead_req = a_req->aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; struct device *dev = ctx->dev; int copy_size, pbuf_length; int req_id = req->req_id; + struct crypto_aead *tfm; + size_t authsize; + u8 *mac_offset; if (ctx->alg_type == SEC_AEAD) copy_size = aead_req->cryptlen + aead_req->assoclen; @@ -709,15 +859,20 @@ static int sec_cipher_pbuf_map(struct sec_ctx *ctx, struct sec_req *req, copy_size = c_req->c_len; pbuf_length = sg_copy_to_buffer(src, sg_nents(src), - qp_ctx->res[req_id].pbuf, - copy_size); + qp_ctx->res[req_id].pbuf, copy_size); if (unlikely(pbuf_length != copy_size)) { dev_err(dev, "copy src data to pbuf error!\n"); return -EINVAL; } + if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { + tfm = crypto_aead_reqtfm(aead_req); + authsize = crypto_aead_authsize(tfm); + mac_offset = qp_ctx->res[req_id].pbuf + copy_size - authsize; + memcpy(a_req->out_mac, mac_offset, authsize); + } - c_req->c_in_dma = qp_ctx->res[req_id].pbuf_dma; - c_req->c_out_dma = c_req->c_in_dma; + req->in_dma = qp_ctx->res[req_id].pbuf_dma; + c_req->c_out_dma = req->in_dma; return 0; } @@ -728,7 +883,6 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, struct aead_request *aead_req = req->aead_req.aead_req; struct sec_cipher_req *c_req = &req->c_req; struct sec_qp_ctx *qp_ctx = req->qp_ctx; - struct device *dev = ctx->dev; int copy_size, pbuf_length; int req_id = req->req_id; @@ -738,10 +892,29 @@ static void sec_cipher_pbuf_unmap(struct sec_ctx *ctx, struct sec_req *req, copy_size = c_req->c_len; pbuf_length = sg_copy_from_buffer(dst, sg_nents(dst), - qp_ctx->res[req_id].pbuf, - copy_size); + qp_ctx->res[req_id].pbuf, copy_size); if (unlikely(pbuf_length != copy_size)) - dev_err(dev, "copy pbuf data to dst error!\n"); + dev_err(ctx->dev, "copy pbuf data to dst error!\n"); +} + +static int sec_aead_mac_init(struct sec_aead_req *req) +{ + struct aead_request *aead_req = req->aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); + u8 *mac_out = req->out_mac; + struct scatterlist *sgl = aead_req->src; + size_t copy_size; + off_t skip_size; + + /* Copy input mac */ + skip_size = aead_req->assoclen + aead_req->cryptlen - authsize; + copy_size = sg_pcopy_to_buffer(sgl, sg_nents(sgl), mac_out, + authsize, skip_size); + if (unlikely(copy_size != authsize)) + return -EINVAL; + + return 0; } static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, @@ -755,37 +928,48 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, int ret; if (req->use_pbuf) { - ret = sec_cipher_pbuf_map(ctx, req, src); c_req->c_ivin = res->pbuf + SEC_PBUF_IV_OFFSET; c_req->c_ivin_dma = res->pbuf_dma + SEC_PBUF_IV_OFFSET; if (ctx->alg_type == SEC_AEAD) { + a_req->a_ivin = res->a_ivin; + a_req->a_ivin_dma = res->a_ivin_dma; a_req->out_mac = res->pbuf + SEC_PBUF_MAC_OFFSET; a_req->out_mac_dma = res->pbuf_dma + SEC_PBUF_MAC_OFFSET; } + ret = sec_cipher_pbuf_map(ctx, req, src); return ret; } c_req->c_ivin = res->c_ivin; c_req->c_ivin_dma = res->c_ivin_dma; if (ctx->alg_type == SEC_AEAD) { + a_req->a_ivin = res->a_ivin; + a_req->a_ivin_dma = res->a_ivin_dma; a_req->out_mac = res->out_mac; a_req->out_mac_dma = res->out_mac_dma; } - c_req->c_in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, - qp_ctx->c_in_pool, - req->req_id, - &c_req->c_in_dma); - - if (IS_ERR(c_req->c_in)) { + req->in = hisi_acc_sg_buf_map_to_hw_sgl(dev, src, + qp_ctx->c_in_pool, + req->req_id, + &req->in_dma); + if (IS_ERR(req->in)) { dev_err(dev, "fail to dma map input sgl buffers!\n"); - return PTR_ERR(c_req->c_in); + return PTR_ERR(req->in); + } + + if (!c_req->encrypt && ctx->alg_type == SEC_AEAD) { + ret = sec_aead_mac_init(a_req); + if (unlikely(ret)) { + dev_err(dev, "fail to init mac data for ICV!\n"); + return ret; + } } if (dst == src) { - c_req->c_out = c_req->c_in; - c_req->c_out_dma = c_req->c_in_dma; + c_req->c_out = req->in; + c_req->c_out_dma = req->in_dma; } else { c_req->c_out = hisi_acc_sg_buf_map_to_hw_sgl(dev, dst, qp_ctx->c_out_pool, @@ -794,7 +978,7 @@ static int sec_cipher_map(struct sec_ctx *ctx, struct sec_req *req, if (IS_ERR(c_req->c_out)) { dev_err(dev, "fail to dma map output sgl buffers!\n"); - hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); + hisi_acc_sg_buf_unmap(dev, src, req->in); return PTR_ERR(c_req->c_out); } } @@ -812,7 +996,7 @@ static void sec_cipher_unmap(struct sec_ctx *ctx, struct sec_req *req, sec_cipher_pbuf_unmap(ctx, req, dst); } else { if (dst != src) - hisi_acc_sg_buf_unmap(dev, src, c_req->c_in); + hisi_acc_sg_buf_unmap(dev, src, req->in); hisi_acc_sg_buf_unmap(dev, dst, c_req->c_out); } @@ -883,6 +1067,28 @@ static int sec_aead_auth_set_key(struct sec_auth_ctx *ctx, return 0; } +static int sec_aead_setauthsize(struct crypto_aead *aead, unsigned int authsize) +{ + struct crypto_tfm *tfm = crypto_aead_tfm(aead); + struct sec_ctx *ctx = crypto_tfm_ctx(tfm); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + + if (unlikely(a_ctx->fallback_aead_tfm)) + return crypto_aead_setauthsize(a_ctx->fallback_aead_tfm, authsize); + + return 0; +} + +static int sec_aead_fallback_setkey(struct sec_auth_ctx *a_ctx, + struct crypto_aead *tfm, const u8 *key, + unsigned int keylen) +{ + crypto_aead_clear_flags(a_ctx->fallback_aead_tfm, CRYPTO_TFM_REQ_MASK); + crypto_aead_set_flags(a_ctx->fallback_aead_tfm, + crypto_aead_get_flags(tfm) & CRYPTO_TFM_REQ_MASK); + return crypto_aead_setkey(a_ctx->fallback_aead_tfm, key, keylen); +} + static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, const u32 keylen, const enum sec_hash_alg a_alg, const enum sec_calg c_alg, @@ -891,6 +1097,7 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, { struct sec_ctx *ctx = crypto_aead_ctx(tfm); struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; struct device *dev = ctx->dev; struct crypto_authenc_keys keys; int ret; @@ -900,6 +1107,23 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, ctx->a_ctx.mac_len = mac_len; c_ctx->c_mode = c_mode; + if (c_mode == SEC_CMODE_CCM || c_mode == SEC_CMODE_GCM) { + ret = sec_skcipher_aes_sm4_setkey(c_ctx, keylen, c_mode); + if (ret) { + dev_err(dev, "set sec aes ccm cipher key err!\n"); + return ret; + } + memcpy(c_ctx->c_key, key, keylen); + + if (unlikely(a_ctx->fallback_aead_tfm)) { + ret = sec_aead_fallback_setkey(a_ctx, tfm, key, keylen); + if (ret) + return ret; + } + + return 0; + } + if (crypto_authenc_extractkeys(&keys, key, keylen)) goto bad_key; @@ -915,6 +1139,12 @@ static int sec_aead_setkey(struct crypto_aead *tfm, const u8 *key, goto bad_key; } + if ((ctx->a_ctx.mac_len & SEC_SQE_LEN_RATE_MASK) || + (ctx->a_ctx.a_key_len & SEC_SQE_LEN_RATE_MASK)) { + dev_err(dev, "MAC or AUTH key length error!\n"); + goto bad_key; + } + return 0; bad_key: @@ -936,6 +1166,14 @@ GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha256, SEC_A_HMAC_SHA256, SEC_CALG_AES, SEC_HMAC_SHA256_MAC, SEC_CMODE_CBC) GEN_SEC_AEAD_SETKEY_FUNC(aes_cbc_sha512, SEC_A_HMAC_SHA512, SEC_CALG_AES, SEC_HMAC_SHA512_MAC, SEC_CMODE_CBC) +GEN_SEC_AEAD_SETKEY_FUNC(aes_ccm, 0, SEC_CALG_AES, + SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(aes_gcm, 0, SEC_CALG_AES, + SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_ccm, 0, SEC_CALG_SM4, + SEC_HMAC_CCM_MAC, SEC_CMODE_CCM) +GEN_SEC_AEAD_SETKEY_FUNC(sm4_gcm, 0, SEC_CALG_SM4, + SEC_HMAC_GCM_MAC, SEC_CMODE_GCM) static int sec_aead_sgl_map(struct sec_ctx *ctx, struct sec_req *req) { @@ -998,7 +1236,7 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) sec_sqe->type2.c_key_addr = cpu_to_le64(c_ctx->c_key_dma); sec_sqe->type2.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); - sec_sqe->type2.data_src_addr = cpu_to_le64(c_req->c_in_dma); + sec_sqe->type2.data_src_addr = cpu_to_le64(req->in_dma); sec_sqe->type2.data_dst_addr = cpu_to_le64(c_req->c_out_dma); sec_sqe->type2.icvw_kmode |= cpu_to_le16(((u16)c_ctx->c_mode) << @@ -1014,29 +1252,86 @@ static int sec_skcipher_bd_fill(struct sec_ctx *ctx, struct sec_req *req) cipher = SEC_CIPHER_DEC << SEC_CIPHER_OFFSET; sec_sqe->type_cipher_auth = bd_type | cipher; - if (req->use_pbuf) + /* Set destination and source address type */ + if (req->use_pbuf) { sa_type = SEC_PBUF << SEC_SRC_SGL_OFFSET; - else + da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; + } else { sa_type = SEC_SGL << SEC_SRC_SGL_OFFSET; + da_type = SEC_SGL << SEC_DST_SGL_OFFSET; + } + + sec_sqe->sdm_addr_type |= da_type; scene = SEC_COMM_SCENE << SEC_SCENE_OFFSET; - if (c_req->c_in_dma != c_req->c_out_dma) + if (req->in_dma != c_req->c_out_dma) de = 0x1 << SEC_DE_OFFSET; sec_sqe->sds_sa_type = (de | scene | sa_type); - /* Just set DST address type */ - if (req->use_pbuf) - da_type = SEC_PBUF << SEC_DST_SGL_OFFSET; - else - da_type = SEC_SGL << SEC_DST_SGL_OFFSET; - sec_sqe->sdm_addr_type |= da_type; - sec_sqe->type2.clen_ivhlen |= cpu_to_le32(c_req->c_len); sec_sqe->type2.tag = cpu_to_le16((u16)req->req_id); return 0; } +static int sec_skcipher_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct sec_cipher_req *c_req = &req->c_req; + u32 bd_param = 0; + u16 cipher; + + memset(sec_sqe3, 0, sizeof(struct sec_sqe3)); + + sec_sqe3->c_key_addr = cpu_to_le64(c_ctx->c_key_dma); + sec_sqe3->no_scene.c_ivin_addr = cpu_to_le64(c_req->c_ivin_dma); + sec_sqe3->data_src_addr = cpu_to_le64(req->in_dma); + sec_sqe3->data_dst_addr = cpu_to_le64(c_req->c_out_dma); + + sec_sqe3->c_mode_alg = ((u8)c_ctx->c_alg << SEC_CALG_OFFSET_V3) | + c_ctx->c_mode; + sec_sqe3->c_icv_key |= cpu_to_le16(((u16)c_ctx->c_key_len) << + SEC_CKEY_OFFSET_V3); + + if (c_req->encrypt) + cipher = SEC_CIPHER_ENC; + else + cipher = SEC_CIPHER_DEC; + sec_sqe3->c_icv_key |= cpu_to_le16(cipher); + + if (req->use_pbuf) { + bd_param |= SEC_PBUF << SEC_SRC_SGL_OFFSET_V3; + bd_param |= SEC_PBUF << SEC_DST_SGL_OFFSET_V3; + } else { + bd_param |= SEC_SGL << SEC_SRC_SGL_OFFSET_V3; + bd_param |= SEC_SGL << SEC_DST_SGL_OFFSET_V3; + } + + bd_param |= SEC_COMM_SCENE << SEC_SCENE_OFFSET_V3; + if (req->in_dma != c_req->c_out_dma) + bd_param |= 0x1 << SEC_DE_OFFSET_V3; + + bd_param |= SEC_BD_TYPE3; + sec_sqe3->bd_param = cpu_to_le32(bd_param); + + sec_sqe3->c_len_ivin |= cpu_to_le32(c_req->c_len); + sec_sqe3->tag = cpu_to_le64(req); + + return 0; +} + +/* increment counter (128-bit int) */ +static void ctr_iv_inc(__u8 *counter, __u8 bits, __u32 nums) +{ + do { + --bits; + nums += counter[bits]; + counter[bits] = nums & BITS_MASK; + nums >>= BYTE_BITS; + } while (bits && nums); +} + static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) { struct aead_request *aead_req = req->aead_req.aead_req; @@ -1060,10 +1355,17 @@ static void sec_update_iv(struct sec_req *req, enum sec_alg_type alg_type) cryptlen = aead_req->cryptlen; } - sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, - cryptlen - iv_size); - if (unlikely(sz != iv_size)) - dev_err(req->ctx->dev, "copy output iv error!\n"); + if (req->ctx->c_ctx.c_mode == SEC_CMODE_CBC) { + sz = sg_pcopy_to_buffer(sgl, sg_nents(sgl), iv, iv_size, + cryptlen - iv_size); + if (unlikely(sz != iv_size)) + dev_err(req->ctx->dev, "copy output iv error!\n"); + } else { + sz = cryptlen / iv_size; + if (cryptlen % iv_size) + sz += 1; + ctr_iv_inc(iv, iv_size, sz); + } } static struct sec_req *sec_back_req_clear(struct sec_ctx *ctx, @@ -1094,8 +1396,9 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, sec_free_req_id(req); - /* IV output at encrypto of CBC mode */ - if (!err && ctx->c_ctx.c_mode == SEC_CMODE_CBC && req->c_req.encrypt) + /* IV output at encrypto of CBC/CTR mode */ + if (!err && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || + ctx->c_ctx.c_mode == SEC_CMODE_CTR) && req->c_req.encrypt) sec_update_iv(req, SEC_SKCIPHER); while (1) { @@ -1112,12 +1415,125 @@ static void sec_skcipher_callback(struct sec_ctx *ctx, struct sec_req *req, sk_req->base.complete(&sk_req->base, err); } -static void sec_aead_copy_iv(struct sec_ctx *ctx, struct sec_req *req) +static void set_aead_auth_iv(struct sec_ctx *ctx, struct sec_req *req) +{ + struct aead_request *aead_req = req->aead_req.aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct sec_aead_req *a_req = &req->aead_req; + size_t authsize = ctx->a_ctx.mac_len; + u32 data_size = aead_req->cryptlen; + u8 flage = 0; + u8 cm, cl; + + /* the specification has been checked in aead_iv_demension_check() */ + cl = c_req->c_ivin[0] + 1; + c_req->c_ivin[ctx->c_ctx.ivsize - cl] = 0x00; + memset(&c_req->c_ivin[ctx->c_ctx.ivsize - cl], 0, cl); + c_req->c_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = IV_CTR_INIT; + + /* the last 3bit is L' */ + flage |= c_req->c_ivin[0] & IV_CL_MASK; + + /* the M' is bit3~bit5, the Flags is bit6 */ + cm = (authsize - IV_CM_CAL_NUM) / IV_CM_CAL_NUM; + flage |= cm << IV_CM_OFFSET; + if (aead_req->assoclen) + flage |= 0x01 << IV_FLAGS_OFFSET; + + memcpy(a_req->a_ivin, c_req->c_ivin, ctx->c_ctx.ivsize); + a_req->a_ivin[0] = flage; + + /* + * the last 32bit is counter's initial number, + * but the nonce uses the first 16bit + * the tail 16bit fill with the cipher length + */ + if (!c_req->encrypt) + data_size = aead_req->cryptlen - authsize; + + a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE1] = + data_size & IV_LAST_BYTE_MASK; + data_size >>= IV_BYTE_OFFSET; + a_req->a_ivin[ctx->c_ctx.ivsize - IV_LAST_BYTE2] = + data_size & IV_LAST_BYTE_MASK; +} + +static void sec_aead_set_iv(struct sec_ctx *ctx, struct sec_req *req) { struct aead_request *aead_req = req->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(aead_req); + size_t authsize = crypto_aead_authsize(tfm); struct sec_cipher_req *c_req = &req->c_req; + struct sec_aead_req *a_req = &req->aead_req; memcpy(c_req->c_ivin, aead_req->iv, ctx->c_ctx.ivsize); + + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM) { + /* + * CCM 16Byte Cipher_IV: {1B_Flage,13B_IV,2B_counter}, + * the counter must set to 0x01 + */ + ctx->a_ctx.mac_len = authsize; + /* CCM 16Byte Auth_IV: {1B_AFlage,13B_IV,2B_Ptext_length} */ + set_aead_auth_iv(ctx, req); + } + + /* GCM 12Byte Cipher_IV == Auth_IV */ + if (ctx->c_ctx.c_mode == SEC_CMODE_GCM) { + ctx->a_ctx.mac_len = authsize; + memcpy(a_req->a_ivin, c_req->c_ivin, SEC_AIV_SIZE); + } +} + +static void sec_auth_bd_fill_xcm(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe *sec_sqe) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aq = a_req->aead_req; + + /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ + sec_sqe->type2.icvw_kmode |= cpu_to_le16((u16)ctx->mac_len); + + /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ + sec_sqe->type2.a_key_addr = sec_sqe->type2.c_key_addr; + sec_sqe->type2.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); + sec_sqe->type_cipher_auth |= SEC_NO_AUTH << SEC_AUTH_OFFSET; + + if (dir) + sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; + else + sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; + + sec_sqe->type2.alen_ivllen = cpu_to_le32(aq->assoclen); + sec_sqe->type2.auth_src_offset = cpu_to_le16(0x0); + sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sec_sqe->type2.mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static void sec_auth_bd_fill_xcm_v3(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe3 *sqe3) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct aead_request *aq = a_req->aead_req; + + /* C_ICV_Len is MAC size, 0x4 ~ 0x10 */ + sqe3->c_icv_key |= cpu_to_le16((u16)ctx->mac_len << SEC_MAC_OFFSET_V3); + + /* mode set to CCM/GCM, don't set {A_Alg, AKey_Len, MAC_Len} */ + sqe3->a_key_addr = sqe3->c_key_addr; + sqe3->auth_ivin.a_ivin_addr = cpu_to_le64(a_req->a_ivin_dma); + sqe3->auth_mac_key |= SEC_NO_AUTH; + + if (dir) + sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; + else + sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; + + sqe3->a_len_key = cpu_to_le32(aq->assoclen); + sqe3->auth_src_offset = cpu_to_le16(0x0); + sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); } static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, @@ -1139,13 +1555,13 @@ static void sec_auth_bd_fill_ex(struct sec_auth_ctx *ctx, int dir, sec_sqe->type2.mac_key_alg |= cpu_to_le32((u32)(ctx->a_alg) << SEC_AEAD_ALG_OFFSET); - sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; - - if (dir) + if (dir) { + sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE1 << SEC_AUTH_OFFSET; sec_sqe->sds_sa_type &= SEC_CIPHER_AUTH; - else + } else { + sec_sqe->type_cipher_auth |= SEC_AUTH_TYPE2 << SEC_AUTH_OFFSET; sec_sqe->sds_sa_type |= SEC_AUTH_CIPHER; - + } sec_sqe->type2.alen_ivllen = cpu_to_le32(c_req->c_len + aq->assoclen); sec_sqe->type2.cipher_src_offset = cpu_to_le16((u16)aq->assoclen); @@ -1165,7 +1581,68 @@ static int sec_aead_bd_fill(struct sec_ctx *ctx, struct sec_req *req) return ret; } - sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || + ctx->c_ctx.c_mode == SEC_CMODE_GCM) + sec_auth_bd_fill_xcm(auth_ctx, req->c_req.encrypt, req, sec_sqe); + else + sec_auth_bd_fill_ex(auth_ctx, req->c_req.encrypt, req, sec_sqe); + + return 0; +} + +static void sec_auth_bd_fill_ex_v3(struct sec_auth_ctx *ctx, int dir, + struct sec_req *req, struct sec_sqe3 *sqe3) +{ + struct sec_aead_req *a_req = &req->aead_req; + struct sec_cipher_req *c_req = &req->c_req; + struct aead_request *aq = a_req->aead_req; + + sqe3->a_key_addr = cpu_to_le64(ctx->a_key_dma); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)(ctx->mac_len / + SEC_SQE_LEN_RATE) << SEC_MAC_OFFSET_V3); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)(ctx->a_key_len / + SEC_SQE_LEN_RATE) << SEC_AKEY_OFFSET_V3); + + sqe3->auth_mac_key |= + cpu_to_le32((u32)(ctx->a_alg) << SEC_AUTH_ALG_OFFSET_V3); + + if (dir) { + sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); + sqe3->huk_iv_seq &= SEC_CIPHER_AUTH_V3; + } else { + sqe3->auth_mac_key |= cpu_to_le32((u32)SEC_AUTH_TYPE1); + sqe3->huk_iv_seq |= SEC_AUTH_CIPHER_V3; + } + sqe3->a_len_key = cpu_to_le32(c_req->c_len + aq->assoclen); + + sqe3->cipher_src_offset = cpu_to_le16((u16)aq->assoclen); + + sqe3->mac_addr = cpu_to_le64(a_req->out_mac_dma); +} + +static int sec_aead_bd_fill_v3(struct sec_ctx *ctx, struct sec_req *req) +{ + struct sec_auth_ctx *auth_ctx = &ctx->a_ctx; + struct sec_sqe3 *sec_sqe3 = &req->sec_sqe3; + int ret; + + ret = sec_skcipher_bd_fill_v3(ctx, req); + if (unlikely(ret)) { + dev_err(ctx->dev, "skcipher bd3 fill is error!\n"); + return ret; + } + + if (ctx->c_ctx.c_mode == SEC_CMODE_CCM || + ctx->c_ctx.c_mode == SEC_CMODE_GCM) + sec_auth_bd_fill_xcm_v3(auth_ctx, req->c_req.encrypt, + req, sec_sqe3); + else + sec_auth_bd_fill_ex_v3(auth_ctx, req->c_req.encrypt, + req, sec_sqe3); return 0; } @@ -1254,7 +1731,8 @@ static int sec_process(struct sec_ctx *ctx, struct sec_req *req) goto err_uninit_req; /* Output IV as decrypto */ - if (ctx->c_ctx.c_mode == SEC_CMODE_CBC && !req->c_req.encrypt) + if (!req->c_req.encrypt && (ctx->c_ctx.c_mode == SEC_CMODE_CBC || + ctx->c_ctx.c_mode == SEC_CMODE_CTR)) sec_update_iv(req, ctx->alg_type); ret = ctx->req_op->bd_send(ctx, req); @@ -1296,20 +1774,51 @@ static const struct sec_req_op sec_skcipher_req_ops = { static const struct sec_req_op sec_aead_req_ops = { .buf_map = sec_aead_sgl_map, .buf_unmap = sec_aead_sgl_unmap, - .do_transfer = sec_aead_copy_iv, + .do_transfer = sec_aead_set_iv, .bd_fill = sec_aead_bd_fill, .bd_send = sec_bd_send, .callback = sec_aead_callback, .process = sec_process, }; +static const struct sec_req_op sec_skcipher_req_ops_v3 = { + .buf_map = sec_skcipher_sgl_map, + .buf_unmap = sec_skcipher_sgl_unmap, + .do_transfer = sec_skcipher_copy_iv, + .bd_fill = sec_skcipher_bd_fill_v3, + .bd_send = sec_bd_send, + .callback = sec_skcipher_callback, + .process = sec_process, +}; + +static const struct sec_req_op sec_aead_req_ops_v3 = { + .buf_map = sec_aead_sgl_map, + .buf_unmap = sec_aead_sgl_unmap, + .do_transfer = sec_aead_set_iv, + .bd_fill = sec_aead_bd_fill_v3, + .bd_send = sec_bd_send, + .callback = sec_aead_callback, + .process = sec_process, +}; + static int sec_skcipher_ctx_init(struct crypto_skcipher *tfm) { struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); + int ret; - ctx->req_op = &sec_skcipher_req_ops; + ret = sec_skcipher_init(tfm); + if (ret) + return ret; - return sec_skcipher_init(tfm); + if (ctx->sec->qm.ver < QM_HW_V3) { + ctx->type_supported = SEC_BD_TYPE2; + ctx->req_op = &sec_skcipher_req_ops; + } else { + ctx->type_supported = SEC_BD_TYPE3; + ctx->req_op = &sec_skcipher_req_ops_v3; + } + + return ret; } static void sec_skcipher_ctx_exit(struct crypto_skcipher *tfm) @@ -1325,15 +1834,22 @@ static int sec_aead_init(struct crypto_aead *tfm) crypto_aead_set_reqsize(tfm, sizeof(struct sec_req)); ctx->alg_type = SEC_AEAD; ctx->c_ctx.ivsize = crypto_aead_ivsize(tfm); - if (ctx->c_ctx.ivsize > SEC_IV_SIZE) { - dev_err(ctx->dev, "get error aead iv size!\n"); + if (ctx->c_ctx.ivsize < SEC_AIV_SIZE || + ctx->c_ctx.ivsize > SEC_IV_SIZE) { + pr_err("get error aead iv size!\n"); return -EINVAL; } - ctx->req_op = &sec_aead_req_ops; ret = sec_ctx_base_init(ctx); if (ret) return ret; + if (ctx->sec->qm.ver < QM_HW_V3) { + ctx->type_supported = SEC_BD_TYPE2; + ctx->req_op = &sec_aead_req_ops; + } else { + ctx->type_supported = SEC_BD_TYPE3; + ctx->req_op = &sec_aead_req_ops_v3; + } ret = sec_auth_init(ctx); if (ret) @@ -1391,6 +1907,41 @@ static void sec_aead_ctx_exit(struct crypto_aead *tfm) sec_aead_exit(tfm); } +static int sec_aead_xcm_ctx_init(struct crypto_aead *tfm) +{ + struct aead_alg *alg = crypto_aead_alg(tfm); + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + const char *aead_name = alg->base.cra_name; + int ret; + + ret = sec_aead_init(tfm); + if (ret) { + dev_err(ctx->dev, "hisi_sec2: aead xcm init error!\n"); + return ret; + } + + a_ctx->fallback_aead_tfm = crypto_alloc_aead(aead_name, 0, + CRYPTO_ALG_NEED_FALLBACK | + CRYPTO_ALG_ASYNC); + if (IS_ERR(a_ctx->fallback_aead_tfm)) { + dev_err(ctx->dev, "aead driver alloc fallback tfm error!\n"); + sec_aead_exit(tfm); + return PTR_ERR(a_ctx->fallback_aead_tfm); + } + a_ctx->fallback = false; + + return 0; +} + +static void sec_aead_xcm_ctx_exit(struct crypto_aead *tfm) +{ + struct sec_ctx *ctx = crypto_aead_ctx(tfm); + + crypto_free_aead(ctx->a_ctx.fallback_aead_tfm); + sec_aead_exit(tfm); +} + static int sec_aead_sha1_ctx_init(struct crypto_aead *tfm) { return sec_aead_ctx_init(tfm, "sha1"); @@ -1429,6 +1980,14 @@ static int sec_skcipher_cryptlen_ckeck(struct sec_ctx *ctx, ret = -EINVAL; } break; + case SEC_CMODE_CFB: + case SEC_CMODE_OFB: + case SEC_CMODE_CTR: + if (unlikely(ctx->sec->qm.ver < QM_HW_V3)) { + dev_err(dev, "skcipher HW version error!\n"); + ret = -EINVAL; + } + break; default: ret = -EINVAL; } @@ -1442,7 +2001,8 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; - if (unlikely(!sk_req->src || !sk_req->dst)) { + if (unlikely(!sk_req->src || !sk_req->dst || + sk_req->cryptlen > MAX_INPUT_DATA_LEN)) { dev_err(dev, "skcipher input param error!\n"); return -EINVAL; } @@ -1468,6 +2028,37 @@ static int sec_skcipher_param_check(struct sec_ctx *ctx, struct sec_req *sreq) return -EINVAL; } +static int sec_skcipher_soft_crypto(struct sec_ctx *ctx, + struct skcipher_request *sreq, bool encrypt) +{ + struct sec_cipher_ctx *c_ctx = &ctx->c_ctx; + struct device *dev = ctx->dev; + int ret; + + SYNC_SKCIPHER_REQUEST_ON_STACK(subreq, c_ctx->fbtfm); + + if (!c_ctx->fbtfm) { + dev_err(dev, "failed to check fallback tfm\n"); + return -EINVAL; + } + + skcipher_request_set_sync_tfm(subreq, c_ctx->fbtfm); + + /* software need sync mode to do crypto */ + skcipher_request_set_callback(subreq, sreq->base.flags, + NULL, NULL); + skcipher_request_set_crypt(subreq, sreq->src, sreq->dst, + sreq->cryptlen, sreq->iv); + if (encrypt) + ret = crypto_skcipher_encrypt(subreq); + else + ret = crypto_skcipher_decrypt(subreq); + + skcipher_request_zero(subreq); + + return ret; +} + static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) { struct crypto_skcipher *tfm = crypto_skcipher_reqtfm(sk_req); @@ -1475,8 +2066,11 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) struct sec_ctx *ctx = crypto_skcipher_ctx(tfm); int ret; - if (!sk_req->cryptlen) + if (!sk_req->cryptlen) { + if (ctx->c_ctx.c_mode == SEC_CMODE_XTS) + return -EINVAL; return 0; + } req->flag = sk_req->base.flags; req->c_req.sk_req = sk_req; @@ -1487,6 +2081,9 @@ static int sec_skcipher_crypto(struct skcipher_request *sk_req, bool encrypt) if (unlikely(ret)) return -EINVAL; + if (unlikely(ctx->c_ctx.fallback)) + return sec_skcipher_soft_crypto(ctx, sk_req, encrypt); + return ctx->req_op->process(ctx, req); } @@ -1507,7 +2104,9 @@ static int sec_skcipher_decrypt(struct skcipher_request *sk_req) .cra_name = sec_cra_name,\ .cra_driver_name = "hisi_sec_"sec_cra_name,\ .cra_priority = SEC_PRIORITY,\ - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ + .cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_ALLOCATES_MEMORY |\ + CRYPTO_ALG_NEED_FALLBACK,\ .cra_blocksize = blk_size,\ .cra_ctxsize = sizeof(struct sec_ctx),\ .cra_module = THIS_MODULE,\ @@ -1541,11 +2140,11 @@ static struct skcipher_alg sec_skciphers[] = { AES_BLOCK_SIZE, AES_BLOCK_SIZE) SEC_SKCIPHER_ALG("ecb(des3_ede)", sec_setkey_3des_ecb, - SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, 0) SEC_SKCIPHER_ALG("cbc(des3_ede)", sec_setkey_3des_cbc, - SEC_DES3_2KEY_SIZE, SEC_DES3_3KEY_SIZE, + SEC_DES3_3KEY_SIZE, SEC_DES3_3KEY_SIZE, DES3_EDE_BLOCK_SIZE, DES3_EDE_BLOCK_SIZE) SEC_SKCIPHER_ALG("xts(sm4)", sec_setkey_sm4_xts, @@ -1557,6 +2156,90 @@ static struct skcipher_alg sec_skciphers[] = { AES_BLOCK_SIZE, AES_BLOCK_SIZE) }; +static struct skcipher_alg sec_skciphers_v3[] = { + SEC_SKCIPHER_ALG("ofb(aes)", sec_setkey_aes_ofb, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("cfb(aes)", sec_setkey_aes_cfb, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("ctr(aes)", sec_setkey_aes_ctr, + AES_MIN_KEY_SIZE, AES_MAX_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("ofb(sm4)", sec_setkey_sm4_ofb, + AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("cfb(sm4)", sec_setkey_sm4_cfb, + AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) + + SEC_SKCIPHER_ALG("ctr(sm4)", sec_setkey_sm4_ctr, + AES_MIN_KEY_SIZE, AES_MIN_KEY_SIZE, + SEC_MIN_BLOCK_SZ, AES_BLOCK_SIZE) +}; + +static int aead_iv_demension_check(struct aead_request *aead_req) +{ + u8 cl; + + cl = aead_req->iv[0] + 1; + if (cl < IV_CL_MIN || cl > IV_CL_MAX) + return -EINVAL; + + if (cl < IV_CL_MID && aead_req->cryptlen >> (BYTE_BITS * cl)) + return -EOVERFLOW; + + return 0; +} + +static int sec_aead_spec_check(struct sec_ctx *ctx, struct sec_req *sreq) +{ + struct aead_request *req = sreq->aead_req.aead_req; + struct crypto_aead *tfm = crypto_aead_reqtfm(req); + size_t authsize = crypto_aead_authsize(tfm); + u8 c_mode = ctx->c_ctx.c_mode; + struct device *dev = ctx->dev; + int ret; + + if (unlikely(req->cryptlen + req->assoclen > MAX_INPUT_DATA_LEN || + req->assoclen > SEC_MAX_AAD_LEN)) { + dev_err(dev, "aead input spec error!\n"); + return -EINVAL; + } + + if (unlikely((c_mode == SEC_CMODE_GCM && authsize < DES_BLOCK_SIZE) || + (c_mode == SEC_CMODE_CCM && (authsize < MIN_MAC_LEN || + authsize & MAC_LEN_MASK)))) { + dev_err(dev, "aead input mac length error!\n"); + return -EINVAL; + } + + if (c_mode == SEC_CMODE_CCM) { + ret = aead_iv_demension_check(req); + if (ret) { + dev_err(dev, "aead input iv param error!\n"); + return ret; + } + } + + if (sreq->c_req.encrypt) + sreq->c_req.c_len = req->cryptlen; + else + sreq->c_req.c_len = req->cryptlen - authsize; + if (c_mode == SEC_CMODE_CBC) { + if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { + dev_err(dev, "aead crypto length error!\n"); + return -EINVAL; + } + } + + return 0; +} + static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) { struct aead_request *req = sreq->aead_req.aead_req; @@ -1565,34 +2248,61 @@ static int sec_aead_param_check(struct sec_ctx *ctx, struct sec_req *sreq) struct device *dev = ctx->dev; u8 c_alg = ctx->c_ctx.c_alg; - if (unlikely(!req->src || !req->dst || !req->cryptlen || - req->assoclen > SEC_MAX_AAD_LEN)) { + if (unlikely(!req->src || !req->dst)) { dev_err(dev, "aead input param error!\n"); return -EINVAL; } + if (ctx->sec->qm.ver == QM_HW_V2) { + if (unlikely(!req->cryptlen || (!sreq->c_req.encrypt && + req->cryptlen <= authsize))) { + dev_err(dev, "Kunpeng920 not support 0 length!\n"); + ctx->a_ctx.fallback = true; + return -EINVAL; + } + } + + /* Support AES or SM4 */ + if (unlikely(c_alg != SEC_CALG_AES && c_alg != SEC_CALG_SM4)) { + dev_err(dev, "aead crypto alg error!\n"); + return -EINVAL; + } + + if (unlikely(sec_aead_spec_check(ctx, sreq))) + return -EINVAL; + if (ctx->pbuf_supported && (req->cryptlen + req->assoclen) <= SEC_PBUF_SZ) sreq->use_pbuf = true; else sreq->use_pbuf = false; - /* Support AES only */ - if (unlikely(c_alg != SEC_CALG_AES)) { - dev_err(dev, "aead crypto alg error!\n"); - return -EINVAL; - } - if (sreq->c_req.encrypt) - sreq->c_req.c_len = req->cryptlen; - else - sreq->c_req.c_len = req->cryptlen - authsize; + return 0; +} - if (unlikely(sreq->c_req.c_len & (AES_BLOCK_SIZE - 1))) { - dev_err(dev, "aead crypto length error!\n"); +static int sec_aead_soft_crypto(struct sec_ctx *ctx, + struct aead_request *aead_req, + bool encrypt) +{ + struct aead_request *subreq = aead_request_ctx(aead_req); + struct sec_auth_ctx *a_ctx = &ctx->a_ctx; + struct device *dev = ctx->dev; + + /* Kunpeng920 aead mode not support input 0 size */ + if (!a_ctx->fallback_aead_tfm) { + dev_err(dev, "aead fallback tfm is NULL!\n"); return -EINVAL; } - return 0; + aead_request_set_tfm(subreq, a_ctx->fallback_aead_tfm); + aead_request_set_callback(subreq, aead_req->base.flags, + aead_req->base.complete, aead_req->base.data); + aead_request_set_crypt(subreq, aead_req->src, aead_req->dst, + aead_req->cryptlen, aead_req->iv); + aead_request_set_ad(subreq, aead_req->assoclen); + + return encrypt ? crypto_aead_encrypt(subreq) : + crypto_aead_decrypt(subreq); } static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) @@ -1608,8 +2318,11 @@ static int sec_aead_crypto(struct aead_request *a_req, bool encrypt) req->ctx = ctx; ret = sec_aead_param_check(ctx, req); - if (unlikely(ret)) + if (unlikely(ret)) { + if (ctx->a_ctx.fallback) + return sec_aead_soft_crypto(ctx, a_req, encrypt); return -EINVAL; + } return ctx->req_op->process(ctx, req); } @@ -1624,14 +2337,16 @@ static int sec_aead_decrypt(struct aead_request *a_req) return sec_aead_crypto(a_req, false); } -#define SEC_AEAD_GEN_ALG(sec_cra_name, sec_set_key, ctx_init,\ +#define SEC_AEAD_ALG(sec_cra_name, sec_set_key, ctx_init,\ ctx_exit, blk_size, iv_size, max_authsize)\ {\ .base = {\ .cra_name = sec_cra_name,\ .cra_driver_name = "hisi_sec_"sec_cra_name,\ .cra_priority = SEC_PRIORITY,\ - .cra_flags = CRYPTO_ALG_ASYNC | CRYPTO_ALG_ALLOCATES_MEMORY,\ + .cra_flags = CRYPTO_ALG_ASYNC |\ + CRYPTO_ALG_ALLOCATES_MEMORY |\ + CRYPTO_ALG_NEED_FALLBACK,\ .cra_blocksize = blk_size,\ .cra_ctxsize = sizeof(struct sec_ctx),\ .cra_module = THIS_MODULE,\ @@ -1639,28 +2354,46 @@ static int sec_aead_decrypt(struct aead_request *a_req) .init = ctx_init,\ .exit = ctx_exit,\ .setkey = sec_set_key,\ + .setauthsize = sec_aead_setauthsize,\ .decrypt = sec_aead_decrypt,\ .encrypt = sec_aead_encrypt,\ .ivsize = iv_size,\ .maxauthsize = max_authsize,\ } -#define SEC_AEAD_ALG(algname, keyfunc, aead_init, blksize, ivsize, authsize)\ - SEC_AEAD_GEN_ALG(algname, keyfunc, aead_init,\ - sec_aead_ctx_exit, blksize, ivsize, authsize) - static struct aead_alg sec_aeads[] = { SEC_AEAD_ALG("authenc(hmac(sha1),cbc(aes))", sec_setkey_aes_cbc_sha1, sec_aead_sha1_ctx_init, - AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), + sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA1_DIGEST_SIZE), SEC_AEAD_ALG("authenc(hmac(sha256),cbc(aes))", sec_setkey_aes_cbc_sha256, sec_aead_sha256_ctx_init, - AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), + sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA256_DIGEST_SIZE), SEC_AEAD_ALG("authenc(hmac(sha512),cbc(aes))", sec_setkey_aes_cbc_sha512, sec_aead_sha512_ctx_init, - AES_BLOCK_SIZE, AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + sec_aead_ctx_exit, AES_BLOCK_SIZE, + AES_BLOCK_SIZE, SHA512_DIGEST_SIZE), + + SEC_AEAD_ALG("ccm(aes)", sec_setkey_aes_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + AES_BLOCK_SIZE, AES_BLOCK_SIZE), + + SEC_AEAD_ALG("gcm(aes)", sec_setkey_aes_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + SEC_AIV_SIZE, AES_BLOCK_SIZE) +}; + +static struct aead_alg sec_aeads_v3[] = { + SEC_AEAD_ALG("ccm(sm4)", sec_setkey_sm4_ccm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + AES_BLOCK_SIZE, AES_BLOCK_SIZE), + + SEC_AEAD_ALG("gcm(sm4)", sec_setkey_sm4_gcm, sec_aead_xcm_ctx_init, + sec_aead_xcm_ctx_exit, SEC_MIN_BLOCK_SZ, + SEC_AIV_SIZE, AES_BLOCK_SIZE) }; int sec_register_to_crypto(struct hisi_qm *qm) @@ -1673,16 +2406,45 @@ int sec_register_to_crypto(struct hisi_qm *qm) if (ret) return ret; + if (qm->ver > QM_HW_V2) { + ret = crypto_register_skciphers(sec_skciphers_v3, + ARRAY_SIZE(sec_skciphers_v3)); + if (ret) + goto reg_skcipher_fail; + } + ret = crypto_register_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); if (ret) - crypto_unregister_skciphers(sec_skciphers, - ARRAY_SIZE(sec_skciphers)); + goto reg_aead_fail; + if (qm->ver > QM_HW_V2) { + ret = crypto_register_aeads(sec_aeads_v3, ARRAY_SIZE(sec_aeads_v3)); + if (ret) + goto reg_aead_v3_fail; + } + return ret; + +reg_aead_v3_fail: + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); +reg_aead_fail: + if (qm->ver > QM_HW_V2) + crypto_unregister_skciphers(sec_skciphers_v3, + ARRAY_SIZE(sec_skciphers_v3)); +reg_skcipher_fail: + crypto_unregister_skciphers(sec_skciphers, + ARRAY_SIZE(sec_skciphers)); return ret; } void sec_unregister_from_crypto(struct hisi_qm *qm) { + if (qm->ver > QM_HW_V2) + crypto_unregister_aeads(sec_aeads_v3, + ARRAY_SIZE(sec_aeads_v3)); + crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); + + if (qm->ver > QM_HW_V2) + crypto_unregister_skciphers(sec_skciphers_v3, + ARRAY_SIZE(sec_skciphers_v3)); crypto_unregister_skciphers(sec_skciphers, ARRAY_SIZE(sec_skciphers)); - crypto_unregister_aeads(sec_aeads, ARRAY_SIZE(sec_aeads)); } diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.h b/drivers/crypto/hisilicon/sec2/sec_crypto.h index 9c78edac56a4..9f71c358a6d3 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.h +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.h @@ -4,9 +4,11 @@ #ifndef __HISI_SEC_V2_CRYPTO_H #define __HISI_SEC_V2_CRYPTO_H +#define SEC_AIV_SIZE 12 #define SEC_IV_SIZE 24 #define SEC_MAX_KEY_SIZE 64 #define SEC_COMM_SCENE 0 +#define SEC_MIN_BLOCK_SZ 1 enum sec_calg { SEC_CALG_3DES = 0x1, @@ -21,6 +23,11 @@ enum sec_hash_alg { }; enum sec_mac_len { + SEC_HMAC_CCM_MAC = 16, + SEC_HMAC_GCM_MAC = 16, + SEC_SM3_MAC = 32, + SEC_HMAC_SM3_MAC = 32, + SEC_HMAC_MD5_MAC = 16, SEC_HMAC_SHA1_MAC = 20, SEC_HMAC_SHA256_MAC = 32, SEC_HMAC_SHA512_MAC = 64, @@ -29,7 +36,11 @@ enum sec_mac_len { enum sec_cmode { SEC_CMODE_ECB = 0x0, SEC_CMODE_CBC = 0x1, + SEC_CMODE_CFB = 0x2, + SEC_CMODE_OFB = 0x3, SEC_CMODE_CTR = 0x4, + SEC_CMODE_CCM = 0x5, + SEC_CMODE_GCM = 0x6, SEC_CMODE_XTS = 0x7, }; @@ -44,6 +55,7 @@ enum sec_ckey_type { enum sec_bd_type { SEC_BD_TYPE1 = 0x1, SEC_BD_TYPE2 = 0x2, + SEC_BD_TYPE3 = 0x3, }; enum sec_auth { @@ -63,6 +75,24 @@ enum sec_addr_type { SEC_PRP = 0x2, }; +struct bd_status { + u64 tag; + u8 done; + u8 err_type; + u16 flag; + u16 icv; +}; + +enum { + AUTHPAD_PAD, + AUTHPAD_NOPAD, +}; + +enum { + AIGEN_GEN, + AIGEN_NOGEN, +}; + struct sec_sqe_type2 { /* * mac_len: 0~4 bits @@ -209,6 +239,169 @@ struct sec_sqe { struct sec_sqe_type2 type2; }; +struct bd3_auth_ivin { + __le64 a_ivin_addr; + __le32 rsvd0; + __le32 rsvd1; +} __packed __aligned(4); + +struct bd3_skip_data { + __le32 rsvd0; + + /* + * gran_num: 0~15 bits + * reserved: 16~31 bits + */ + __le32 gran_num; + + /* + * src_skip_data_len: 0~24 bits + * reserved: 25~31 bits + */ + __le32 src_skip_data_len; + + /* + * dst_skip_data_len: 0~24 bits + * reserved: 25~31 bits + */ + __le32 dst_skip_data_len; +}; + +struct bd3_stream_scene { + __le64 c_ivin_addr; + __le64 long_a_data_len; + + /* + * auth_pad: 0~1 bits + * stream_protocol: 2~4 bits + * reserved: 5~7 bits + */ + __u8 stream_auth_pad; + __u8 plaintext_type; + __le16 pad_len_1p3; +} __packed __aligned(4); + +struct bd3_no_scene { + __le64 c_ivin_addr; + __le32 rsvd0; + __le32 rsvd1; + __le32 rsvd2; +} __packed __aligned(4); + +struct bd3_check_sum { + __u8 rsvd0; + __u8 hac_sva_status; + __le16 check_sum_i; +}; + +struct bd3_tls_type_back { + __u8 tls_1p3_type_back; + __u8 hac_sva_status; + __le16 pad_len_1p3_back; +}; + +struct sec_sqe3 { + /* + * type: 0~3 bit + * bd_invalid: 4 bit + * scene: 5~8 bit + * de: 9~10 bit + * src_addr_type: 11~13 bit + * dst_addr_type: 14~16 bit + * mac_addr_type: 17~19 bit + * reserved: 20~31 bits + */ + __le32 bd_param; + + /* + * cipher: 0~1 bits + * ci_gen: 2~3 bit + * c_icv_len: 4~9 bit + * c_width: 10~12 bits + * c_key_len: 13~15 bits + */ + __le16 c_icv_key; + + /* + * c_mode : 0~3 bits + * c_alg : 4~7 bits + */ + __u8 c_mode_alg; + + /* + * nonce_len : 0~3 bits + * huk : 4 bits + * cal_iv_addr_en : 5 bits + * seq : 6 bits + * reserved : 7 bits + */ + __u8 huk_iv_seq; + + __le64 tag; + __le64 data_src_addr; + __le64 a_key_addr; + union { + struct bd3_auth_ivin auth_ivin; + struct bd3_skip_data skip_data; + }; + + __le64 c_key_addr; + + /* + * auth: 0~1 bits + * ai_gen: 2~3 bits + * mac_len: 4~8 bits + * akey_len: 9~14 bits + * a_alg: 15~20 bits + * key_sel: 21~24 bits + * updata_key: 25 bits + * reserved: 26~31 bits + */ + __le32 auth_mac_key; + __le32 salt; + __le16 auth_src_offset; + __le16 cipher_src_offset; + + /* + * auth_len: 0~23 bit + * auth_key_offset: 24~31 bits + */ + __le32 a_len_key; + + /* + * cipher_len: 0~23 bit + * auth_ivin_offset: 24~31 bits + */ + __le32 c_len_ivin; + __le64 data_dst_addr; + __le64 mac_addr; + union { + struct bd3_stream_scene stream_scene; + struct bd3_no_scene no_scene; + }; + + /* + * done: 0 bit + * icv: 1~3 bit + * csc: 4~6 bit + * flag: 7~10 bit + * reserved: 11~15 bit + */ + __le16 done_flag; + __u8 error_type; + __u8 warning_type; + union { + __le32 mac_i; + __le32 kek_key_addr_l; + }; + union { + __le32 kek_key_addr_h; + struct bd3_check_sum check_sum; + struct bd3_tls_type_back tls_type_back; + }; + __le32 counter; +} __packed __aligned(4); + int sec_register_to_crypto(struct hisi_qm *qm); void sec_unregister_from_crypto(struct hisi_qm *qm); #endif diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index 6f0062d4408c..d120ce3e34ed 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -52,6 +52,7 @@ #define SEC_RAS_CE_ENB_MSK 0x88 #define SEC_RAS_FE_ENB_MSK 0x0 #define SEC_RAS_NFE_ENB_MSK 0x7c177 +#define SEC_OOO_SHUTDOWN_SEL 0x301014 #define SEC_RAS_DISABLE 0x0 #define SEC_MEM_START_INIT_REG 0x301100 #define SEC_MEM_INIT_DONE_REG 0x301104 @@ -84,6 +85,12 @@ #define SEC_USER1_SMMU_MASK (~SEC_USER1_SVA_SET) #define SEC_CORE_INT_STATUS_M_ECC BIT(2) +#define SEC_PREFETCH_CFG 0x301130 +#define SEC_SVA_TRANS 0x301EC4 +#define SEC_PREFETCH_ENABLE (~(BIT(0) | BIT(1) | BIT(11))) +#define SEC_PREFETCH_DISABLE BIT(1) +#define SEC_SVA_DISABLE_READY (BIT(7) | BIT(11)) + #define SEC_DELAY_10_US 10 #define SEC_POLL_TIMEOUT_US 1000 #define SEC_DBGFS_VAL_MAX_LEN 20 @@ -91,6 +98,7 @@ #define SEC_SQE_MASK_OFFSET 64 #define SEC_SQE_MASK_LEN 48 +#define SEC_SHAPER_TYPE_RATE 128 struct sec_hw_error { u32 int_msk; @@ -331,6 +339,42 @@ static u8 sec_get_endian(struct hisi_qm *qm) return SEC_64BE; } +static void sec_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val &= SEC_PREFETCH_ENABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_PREFETCH_CFG, + val, !(val & SEC_PREFETCH_DISABLE), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to open sva prefetch\n"); +} + +static void sec_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + val = readl_relaxed(qm->io_base + SEC_PREFETCH_CFG); + val |= SEC_PREFETCH_DISABLE; + writel(val, qm->io_base + SEC_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + SEC_SVA_TRANS, + val, !(val & SEC_SVA_DISABLE_READY), + SEC_DELAY_10_US, SEC_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); +} + static int sec_engine_init(struct hisi_qm *qm) { int ret; @@ -430,53 +474,60 @@ static void sec_debug_regs_clear(struct hisi_qm *qm) hisi_qm_debug_regs_clear(qm); } -static void sec_hw_error_enable(struct hisi_qm *qm) +static void sec_master_ooo_ctrl(struct hisi_qm *qm, bool enable) { - u32 val; + u32 val1, val2; + + val1 = readl(qm->io_base + SEC_CONTROL_REG); + if (enable) { + val1 |= SEC_AXI_SHUTDOWN_ENABLE; + val2 = SEC_RAS_NFE_ENB_MSK; + } else { + val1 &= SEC_AXI_SHUTDOWN_DISABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + SEC_OOO_SHUTDOWN_SEL); + + writel(val1, qm->io_base + SEC_CONTROL_REG); +} +static void sec_hw_error_enable(struct hisi_qm *qm) +{ if (qm->ver == QM_HW_V1) { writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); pci_info(qm->pdev, "V1 not support hw error handle\n"); return; } - val = readl(qm->io_base + SEC_CONTROL_REG); - /* clear SEC hw error source if having */ writel(SEC_CORE_INT_CLEAR, qm->io_base + SEC_CORE_INT_SOURCE); - /* enable SEC hw error interrupts */ - writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); - /* enable RAS int */ writel(SEC_RAS_CE_ENB_MSK, qm->io_base + SEC_RAS_CE_REG); writel(SEC_RAS_FE_ENB_MSK, qm->io_base + SEC_RAS_FE_REG); writel(SEC_RAS_NFE_ENB_MSK, qm->io_base + SEC_RAS_NFE_REG); - /* enable SEC block master OOO when m-bit error occur */ - val = val | SEC_AXI_SHUTDOWN_ENABLE; + /* enable SEC block master OOO when nfe occurs on Kunpeng930 */ + sec_master_ooo_ctrl(qm, true); - writel(val, qm->io_base + SEC_CONTROL_REG); + /* enable SEC hw error interrupts */ + writel(SEC_CORE_INT_ENABLE, qm->io_base + SEC_CORE_INT_MASK); } static void sec_hw_error_disable(struct hisi_qm *qm) { - u32 val; + /* disable SEC hw error interrupts */ + writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); - val = readl(qm->io_base + SEC_CONTROL_REG); + /* disable SEC block master OOO when nfe occurs on Kunpeng930 */ + sec_master_ooo_ctrl(qm, false); /* disable RAS int */ writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_CE_REG); writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_FE_REG); writel(SEC_RAS_DISABLE, qm->io_base + SEC_RAS_NFE_REG); - - /* disable SEC hw error interrupts */ - writel(SEC_CORE_INT_DISABLE, qm->io_base + SEC_CORE_INT_MASK); - - /* disable SEC block master OOO when m-bit error occur */ - val = val & SEC_AXI_SHUTDOWN_DISABLE; - - writel(val, qm->io_base + SEC_CONTROL_REG); } static u32 sec_clear_enable_read(struct sec_debug_file *file) @@ -743,6 +794,8 @@ static const struct hisi_qm_err_ini sec_err_ini = { .clear_dev_hw_err_status = sec_clear_hw_err_status, .log_dev_hw_err = sec_log_hw_error, .open_axi_master_ooo = sec_open_axi_master_ooo, + .open_sva_prefetch = sec_open_sva_prefetch, + .close_sva_prefetch = sec_close_sva_prefetch, .err_info_init = sec_err_info_init, }; @@ -758,6 +811,7 @@ static int sec_pf_probe_init(struct sec_dev *sec) if (ret) return ret; + sec_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); sec_debug_regs_clear(qm); @@ -821,6 +875,7 @@ static void sec_qm_uninit(struct hisi_qm *qm) static int sec_probe_init(struct sec_dev *sec) { + u32 type_rate = SEC_SHAPER_TYPE_RATE; struct hisi_qm *qm = &sec->qm; int ret; @@ -828,6 +883,11 @@ static int sec_probe_init(struct sec_dev *sec) ret = sec_pf_probe_init(sec); if (ret) return ret; + /* enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + qm->type_rate = type_rate; + } } return 0; diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index 2178b40e9f82..f8482ceebf2a 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -68,6 +68,7 @@ #define HZIP_CORE_INT_RAS_CE_ENABLE 0x1 #define HZIP_CORE_INT_RAS_NFE_ENB 0x301164 #define HZIP_CORE_INT_RAS_FE_ENB 0x301168 +#define HZIP_OOO_SHUTDOWN_SEL 0x30120C #define HZIP_CORE_INT_RAS_NFE_ENABLE 0x1FFE #define HZIP_SRAM_ECC_ERR_NUM_SHIFT 16 #define HZIP_SRAM_ECC_ERR_ADDR_SHIFT 24 @@ -96,6 +97,16 @@ #define HZIP_RD_CNT_CLR_CE_EN (HZIP_CNT_CLR_CE_EN | \ HZIP_RO_CNT_CLR_CE_EN) +#define HZIP_PREFETCH_CFG 0x3011B0 +#define HZIP_SVA_TRANS 0x3011C4 +#define HZIP_PREFETCH_ENABLE (~(BIT(26) | BIT(17) | BIT(0))) +#define HZIP_SVA_PREFETCH_DISABLE BIT(26) +#define HZIP_SVA_DISABLE_READY (BIT(26) | BIT(30)) +#define HZIP_SHAPER_RATE_COMPRESS 252 +#define HZIP_SHAPER_RATE_DECOMPRESS 229 +#define HZIP_DELAY_1_US 1 +#define HZIP_POLL_TIMEOUT_US 1000 + static const char hisi_zip_name[] = "hisi_zip"; static struct dentry *hzip_debugfs_root; @@ -262,6 +273,45 @@ int zip_create_qps(struct hisi_qp **qps, int qp_num, int node) return hisi_qm_alloc_qps_node(&zip_devices, qp_num, 0, node, qps); } +static void hisi_zip_open_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + /* Enable prefetch */ + val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); + val &= HZIP_PREFETCH_ENABLE; + writel(val, qm->io_base + HZIP_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_PREFETCH_CFG, + val, !(val & HZIP_SVA_PREFETCH_DISABLE), + HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to open sva prefetch\n"); +} + +static void hisi_zip_close_sva_prefetch(struct hisi_qm *qm) +{ + u32 val; + int ret; + + if (qm->ver < QM_HW_V3) + return; + + val = readl_relaxed(qm->io_base + HZIP_PREFETCH_CFG); + val |= HZIP_SVA_PREFETCH_DISABLE; + writel(val, qm->io_base + HZIP_PREFETCH_CFG); + + ret = readl_relaxed_poll_timeout(qm->io_base + HZIP_SVA_TRANS, + val, !(val & HZIP_SVA_DISABLE_READY), + HZIP_DELAY_1_US, HZIP_POLL_TIMEOUT_US); + if (ret) + pci_err(qm->pdev, "failed to close sva prefetch\n"); +} + static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) { void __iomem *base = qm->io_base; @@ -312,10 +362,27 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) return 0; } -static void hisi_zip_hw_error_enable(struct hisi_qm *qm) +static void hisi_zip_master_ooo_ctrl(struct hisi_qm *qm, bool enable) { - u32 val; + u32 val1, val2; + + val1 = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + if (enable) { + val1 |= HZIP_AXI_SHUTDOWN_ENABLE; + val2 = HZIP_CORE_INT_RAS_NFE_ENABLE; + } else { + val1 &= ~HZIP_AXI_SHUTDOWN_ENABLE; + val2 = 0x0; + } + + if (qm->ver > QM_HW_V2) + writel(val2, qm->io_base + HZIP_OOO_SHUTDOWN_SEL); + writel(val1, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); +} + +static void hisi_zip_hw_error_enable(struct hisi_qm *qm) +{ if (qm->ver == QM_HW_V1) { writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); @@ -333,26 +400,20 @@ static void hisi_zip_hw_error_enable(struct hisi_qm *qm) writel(HZIP_CORE_INT_RAS_NFE_ENABLE, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); + /* enable ZIP block master OOO when nfe occurs on Kunpeng930 */ + hisi_zip_master_ooo_ctrl(qm, true); + /* enable ZIP hw error interrupts */ writel(0, qm->io_base + HZIP_CORE_INT_MASK_REG); - - /* enable ZIP block master OOO when m-bit error occur */ - val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); - val = val | HZIP_AXI_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); } static void hisi_zip_hw_error_disable(struct hisi_qm *qm) { - u32 val; - /* disable ZIP hw error interrupts */ writel(HZIP_CORE_INT_MASK_ALL, qm->io_base + HZIP_CORE_INT_MASK_REG); - /* disable ZIP block master OOO when m-bit error occur */ - val = readl(qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); - val = val & ~HZIP_AXI_SHUTDOWN_ENABLE; - writel(val, qm->io_base + HZIP_SOFT_CTRL_ZIP_CONTROL); + /* disable ZIP block master OOO when nfe occurs on Kunpeng930 */ + hisi_zip_master_ooo_ctrl(qm, false); } static inline struct hisi_qm *file_to_qm(struct ctrl_debug_file *file) @@ -684,6 +745,8 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .log_dev_hw_err = hisi_zip_log_hw_error, .open_axi_master_ooo = hisi_zip_open_axi_master_ooo, .close_axi_master_ooo = hisi_zip_close_axi_master_ooo, + .open_sva_prefetch = hisi_zip_open_sva_prefetch, + .close_sva_prefetch = hisi_zip_close_sva_prefetch, .err_info_init = hisi_zip_err_info_init, }; @@ -702,6 +765,7 @@ static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) qm->err_ini->err_info_init(qm); hisi_zip_set_user_domain_and_cache(qm); + hisi_zip_open_sva_prefetch(qm); hisi_qm_dev_err_init(qm); hisi_zip_debug_regs_clear(qm); @@ -761,6 +825,7 @@ static void hisi_zip_qm_uninit(struct hisi_qm *qm) static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) { + u32 type_rate = HZIP_SHAPER_RATE_COMPRESS; struct hisi_qm *qm = &hisi_zip->qm; int ret; @@ -768,6 +833,14 @@ static int hisi_zip_probe_init(struct hisi_zip *hisi_zip) ret = hisi_zip_pf_probe_init(hisi_zip); if (ret) return ret; + /* enable shaper type 0 */ + if (qm->ver >= QM_HW_V3) { + type_rate |= QM_SHAPER_ENABLE; + + /* ZIP need to enable shaper type 1 */ + type_rate |= HZIP_SHAPER_RATE_DECOMPRESS << QM_SHAPER_TYPE1_OFFSET; + qm->type_rate = type_rate; + } } return 0; |