diff options
170 files changed, 5964 insertions, 4104 deletions
diff --git a/Documentation/ABI/testing/debugfs-hisi-hpre b/Documentation/ABI/testing/debugfs-hisi-hpre index d4e16ef9ac9a..29fb7d5ffc69 100644 --- a/Documentation/ABI/testing/debugfs-hisi-hpre +++ b/Documentation/ABI/testing/debugfs-hisi-hpre @@ -184,3 +184,10 @@ Date: Apr 2020 Contact: linux-crypto@vger.kernel.org Description: Dump the total number of time out requests. Available for both PF and VF, and take no other effect on HPRE. + +What: /sys/kernel/debug/hisi_hpre/<bdf>/cap_regs +Date: Oct 2024 +Contact: linux-crypto@vger.kernel.org +Description: Dump the values of the qm and hpre capability bit registers and + support the query of device specifications to facilitate fault locating. + Available for both PF and VF, and take no other effect on HPRE. diff --git a/Documentation/ABI/testing/debugfs-hisi-sec b/Documentation/ABI/testing/debugfs-hisi-sec index 6c6c9a6e150a..82bf4a0dc7f7 100644 --- a/Documentation/ABI/testing/debugfs-hisi-sec +++ b/Documentation/ABI/testing/debugfs-hisi-sec @@ -157,3 +157,10 @@ Contact: linux-crypto@vger.kernel.org Description: Dump the total number of completed but marked error requests to be received. Available for both PF and VF, and take no other effect on SEC. + +What: /sys/kernel/debug/hisi_sec2/<bdf>/cap_regs +Date: Oct 2024 +Contact: linux-crypto@vger.kernel.org +Description: Dump the values of the qm and sec capability bit registers and + support the query of device specifications to facilitate fault locating. + Available for both PF and VF, and take no other effect on SEC. diff --git a/Documentation/ABI/testing/debugfs-hisi-zip b/Documentation/ABI/testing/debugfs-hisi-zip index a22dd6942219..0abd65d27e9b 100644 --- a/Documentation/ABI/testing/debugfs-hisi-zip +++ b/Documentation/ABI/testing/debugfs-hisi-zip @@ -158,3 +158,10 @@ Contact: linux-crypto@vger.kernel.org Description: Dump the total number of BD type error requests to be received. Available for both PF and VF, and take no other effect on ZIP. + +What: /sys/kernel/debug/hisi_zip/<bdf>/cap_regs +Date: Oct 2024 +Contact: linux-crypto@vger.kernel.org +Description: Dump the values of the qm and zip capability bit registers and + support the query of device specifications to facilitate fault locating. + Available for both PF and VF, and take no other effect on ZIP. diff --git a/Documentation/crypto/api-akcipher.rst b/Documentation/crypto/api-akcipher.rst index 40aa8746e2a1..ca1ecdd4a7d3 100644 --- a/Documentation/crypto/api-akcipher.rst +++ b/Documentation/crypto/api-akcipher.rst @@ -8,10 +8,10 @@ Asymmetric Cipher API --------------------- .. kernel-doc:: include/crypto/akcipher.h - :doc: Generic Public Key API + :doc: Generic Public Key Cipher API .. kernel-doc:: include/crypto/akcipher.h - :functions: crypto_alloc_akcipher crypto_free_akcipher crypto_akcipher_set_pub_key crypto_akcipher_set_priv_key crypto_akcipher_maxsize crypto_akcipher_encrypt crypto_akcipher_decrypt crypto_akcipher_sign crypto_akcipher_verify + :functions: crypto_alloc_akcipher crypto_free_akcipher crypto_akcipher_set_pub_key crypto_akcipher_set_priv_key crypto_akcipher_maxsize crypto_akcipher_encrypt crypto_akcipher_decrypt Asymmetric Cipher Request Handle -------------------------------- diff --git a/Documentation/crypto/api-sig.rst b/Documentation/crypto/api-sig.rst new file mode 100644 index 000000000000..aaec18e26d54 --- /dev/null +++ b/Documentation/crypto/api-sig.rst @@ -0,0 +1,15 @@ +Asymmetric Signature Algorithm Definitions +------------------------------------------ + +.. kernel-doc:: include/crypto/sig.h + :functions: sig_alg + +Asymmetric Signature API +------------------------ + +.. kernel-doc:: include/crypto/sig.h + :doc: Generic Public Key Signature API + +.. kernel-doc:: include/crypto/sig.h + :functions: crypto_alloc_sig crypto_free_sig crypto_sig_set_pubkey crypto_sig_set_privkey crypto_sig_keysize crypto_sig_maxsize crypto_sig_digestsize crypto_sig_sign crypto_sig_verify + diff --git a/Documentation/crypto/api.rst b/Documentation/crypto/api.rst index ff31c30561d4..8b2a90521886 100644 --- a/Documentation/crypto/api.rst +++ b/Documentation/crypto/api.rst @@ -10,4 +10,5 @@ Programming Interface api-digest api-rng api-akcipher + api-sig api-kpp diff --git a/Documentation/crypto/architecture.rst b/Documentation/crypto/architecture.rst index 646c3380a7ed..15dcd62fd22f 100644 --- a/Documentation/crypto/architecture.rst +++ b/Documentation/crypto/architecture.rst @@ -214,6 +214,8 @@ the aforementioned cipher types: - CRYPTO_ALG_TYPE_AKCIPHER Asymmetric cipher +- CRYPTO_ALG_TYPE_SIG Asymmetric signature + - CRYPTO_ALG_TYPE_PCOMPRESS Enhanced version of CRYPTO_ALG_TYPE_COMPRESS allowing for segmented compression / decompression instead of performing the operation on one segment diff --git a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml index e285e382d4ec..c09be97434ac 100644 --- a/Documentation/devicetree/bindings/crypto/qcom-qce.yaml +++ b/Documentation/devicetree/bindings/crypto/qcom-qce.yaml @@ -44,6 +44,7 @@ properties: - items: - enum: + - qcom,sa8775p-qce - qcom,sc7280-qce - qcom,sm6350-qce - qcom,sm8250-qce diff --git a/Documentation/devicetree/bindings/rng/airoha,en7581-trng.yaml b/Documentation/devicetree/bindings/rng/airoha,en7581-trng.yaml new file mode 100644 index 000000000000..dfc6d24ee7d9 --- /dev/null +++ b/Documentation/devicetree/bindings/rng/airoha,en7581-trng.yaml @@ -0,0 +1,38 @@ +# SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rng/airoha,en7581-trng.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: Airoha EN7851 True Random Number Generator + +maintainers: + - Christian Marangi <ansuelsmth@gmail.com> + +properties: + compatible: + const: airoha,en7581-trng + + reg: + maxItems: 1 + + interrupts: + maxItems: 1 + +required: + - compatible + - reg + - interrupts + +additionalProperties: false + +examples: + - | + #include <dt-bindings/interrupt-controller/irq.h> + #include <dt-bindings/interrupt-controller/arm-gic.h> + + rng@1faa1000 { + compatible = "airoha,en7581-trng"; + reg = <0x1faa1000 0x1000>; + interrupts = <GIC_SPI 19 IRQ_TYPE_LEVEL_HIGH>; + }; diff --git a/Documentation/devicetree/bindings/rng/brcm,bcm74110-rng.yaml b/Documentation/devicetree/bindings/rng/brcm,bcm74110-rng.yaml new file mode 100644 index 000000000000..8e89d4a70b53 --- /dev/null +++ b/Documentation/devicetree/bindings/rng/brcm,bcm74110-rng.yaml @@ -0,0 +1,35 @@ +# SPDX-License-Identifier: (GPL-2.0 OR BSD-2-Clause) +%YAML 1.2 +--- +$id: http://devicetree.org/schemas/rng/brcm,bcm74110-rng.yaml# +$schema: http://devicetree.org/meta-schemas/core.yaml# + +title: BCM74110 Random number generator + +description: + Random number generator used on the BCM74110. + +maintainers: + - Markus Mayer <mmayer@broadcom.com> + - Florian Fainelli <florian.fainelli@broadcom.com> + +properties: + compatible: + enum: + - brcm,bcm74110-rng + + reg: + maxItems: 1 + +required: + - compatible + - reg + +additionalProperties: false + +examples: + - | + rng@83ba000 { + compatible = "brcm,bcm74110-rng"; + reg = <0x83ba000 0x14>; + }; diff --git a/Documentation/devicetree/bindings/rng/imx-rng.yaml b/Documentation/devicetree/bindings/rng/imx-rng.yaml index 07f6ff89bcc1..252fa9a41abe 100644 --- a/Documentation/devicetree/bindings/rng/imx-rng.yaml +++ b/Documentation/devicetree/bindings/rng/imx-rng.yaml @@ -14,8 +14,8 @@ properties: oneOf: - const: fsl,imx21-rnga - const: fsl,imx25-rngb + - const: fsl,imx31-rnga - items: - - const: fsl,imx31-rnga - const: fsl,imx21-rnga - items: - enum: diff --git a/Documentation/devicetree/bindings/rng/omap_rng.yaml b/Documentation/devicetree/bindings/rng/inside-secure,safexcel-eip76.yaml index c0ac4f68ea54..0877eb44f9ed 100644 --- a/Documentation/devicetree/bindings/rng/omap_rng.yaml +++ b/Documentation/devicetree/bindings/rng/inside-secure,safexcel-eip76.yaml @@ -1,20 +1,25 @@ # SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) %YAML 1.2 --- -$id: http://devicetree.org/schemas/rng/omap_rng.yaml# +$id: http://devicetree.org/schemas/rng/inside-secure,safexcel-eip76.yaml# $schema: http://devicetree.org/meta-schemas/core.yaml# -title: OMAP SoC and Inside-Secure HWRNG Module +title: Inside-Secure HWRNG Module maintainers: - Jayesh Choudhary <j-choudhary@ti.com> properties: compatible: - enum: - - ti,omap2-rng - - ti,omap4-rng - - inside-secure,safexcel-eip76 + oneOf: + - enum: + - ti,omap2-rng + - ti,omap4-rng + - inside-secure,safexcel-eip76 + - items: + - enum: + - marvell,armada-8k-rng + - const: inside-secure,safexcel-eip76 ti,hwmods: const: rng diff --git a/Documentation/devicetree/bindings/rng/st,stm32-rng.yaml b/Documentation/devicetree/bindings/rng/st,stm32-rng.yaml index 340d01d481d1..7db65f49773b 100644 --- a/Documentation/devicetree/bindings/rng/st,stm32-rng.yaml +++ b/Documentation/devicetree/bindings/rng/st,stm32-rng.yaml @@ -18,12 +18,19 @@ properties: enum: - st,stm32-rng - st,stm32mp13-rng + - st,stm32mp25-rng reg: maxItems: 1 clocks: - maxItems: 1 + minItems: 1 + maxItems: 2 + + clock-names: + items: + - const: core + - const: bus resets: maxItems: 1 @@ -57,6 +64,25 @@ allOf: properties: st,rng-lock-conf: false + - if: + properties: + compatible: + contains: + enum: + - st,stm32-rng + - st,stm32mp13-rng + then: + properties: + clocks: + maxItems: 1 + clock-names: false + else: + properties: + clocks: + minItems: 2 + required: + - clock-names + additionalProperties: false examples: diff --git a/MAINTAINERS b/MAINTAINERS index 82161bc70b51..6b3ef1161f89 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11447,7 +11447,7 @@ Q: https://patchwork.kernel.org/project/linux-dmaengine/list/ F: drivers/dma/ioat* INTEL IAA CRYPTO DRIVER -M: Tom Zanussi <tom.zanussi@linux.intel.com> +M: Kristen Accardi <kristen.c.accardi@intel.com> L: linux-crypto@vger.kernel.org S: Supported F: Documentation/driver-api/crypto/iaa/iaa-crypto.rst diff --git a/arch/arm/crypto/crct10dif-ce-core.S b/arch/arm/crypto/crct10dif-ce-core.S index 46c02c518a30..2bbf2df9c1e2 100644 --- a/arch/arm/crypto/crct10dif-ce-core.S +++ b/arch/arm/crypto/crct10dif-ce-core.S @@ -112,55 +112,120 @@ FOLD_CONST_L .req q10l FOLD_CONST_H .req q10h + /* + * Pairwise long polynomial multiplication of two 16-bit values + * + * { w0, w1 }, { y0, y1 } + * + * by two 64-bit values + * + * { x0, x1, x2, x3, x4, x5, x6, x7 }, { z0, z1, z2, z3, z4, z5, z6, z7 } + * + * where each vector element is a byte, ordered from least to most + * significant. The resulting 80-bit vectors are XOR'ed together. + * + * This can be implemented using 8x8 long polynomial multiplication, by + * reorganizing the input so that each pairwise 8x8 multiplication + * produces one of the terms from the decomposition below, and + * combining the results of each rank and shifting them into place. + * + * Rank + * 0 w0*x0 ^ | y0*z0 ^ + * 1 (w0*x1 ^ w1*x0) << 8 ^ | (y0*z1 ^ y1*z0) << 8 ^ + * 2 (w0*x2 ^ w1*x1) << 16 ^ | (y0*z2 ^ y1*z1) << 16 ^ + * 3 (w0*x3 ^ w1*x2) << 24 ^ | (y0*z3 ^ y1*z2) << 24 ^ + * 4 (w0*x4 ^ w1*x3) << 32 ^ | (y0*z4 ^ y1*z3) << 32 ^ + * 5 (w0*x5 ^ w1*x4) << 40 ^ | (y0*z5 ^ y1*z4) << 40 ^ + * 6 (w0*x6 ^ w1*x5) << 48 ^ | (y0*z6 ^ y1*z5) << 48 ^ + * 7 (w0*x7 ^ w1*x6) << 56 ^ | (y0*z7 ^ y1*z6) << 56 ^ + * 8 w1*x7 << 64 | y1*z7 << 64 + * + * The inputs can be reorganized into + * + * { w0, w0, w0, w0, y0, y0, y0, y0 }, { w1, w1, w1, w1, y1, y1, y1, y1 } + * { x0, x2, x4, x6, z0, z2, z4, z6 }, { x1, x3, x5, x7, z1, z3, z5, z7 } + * + * and after performing 8x8->16 bit long polynomial multiplication of + * each of the halves of the first vector with those of the second one, + * we obtain the following four vectors of 16-bit elements: + * + * a := { w0*x0, w0*x2, w0*x4, w0*x6 }, { y0*z0, y0*z2, y0*z4, y0*z6 } + * b := { w0*x1, w0*x3, w0*x5, w0*x7 }, { y0*z1, y0*z3, y0*z5, y0*z7 } + * c := { w1*x0, w1*x2, w1*x4, w1*x6 }, { y1*z0, y1*z2, y1*z4, y1*z6 } + * d := { w1*x1, w1*x3, w1*x5, w1*x7 }, { y1*z1, y1*z3, y1*z5, y1*z7 } + * + * Results b and c can be XORed together, as the vector elements have + * matching ranks. Then, the final XOR can be pulled forward, and + * applied between the halves of each of the remaining three vectors, + * which are then shifted into place, and XORed together to produce the + * final 80-bit result. + */ + .macro pmull16x64_p8, v16, v64 + vext.8 q11, \v64, \v64, #1 + vld1.64 {q12}, [r4, :128] + vuzp.8 q11, \v64 + vtbl.8 d24, {\v16\()_L-\v16\()_H}, d24 + vtbl.8 d25, {\v16\()_L-\v16\()_H}, d25 + bl __pmull16x64_p8 + veor \v64, q12, q14 + .endm + +__pmull16x64_p8: + vmull.p8 q13, d23, d24 + vmull.p8 q14, d23, d25 + vmull.p8 q15, d22, d24 + vmull.p8 q12, d22, d25 + + veor q14, q14, q15 + veor d24, d24, d25 + veor d26, d26, d27 + veor d28, d28, d29 + vmov.i32 d25, #0 + vmov.i32 d29, #0 + vext.8 q12, q12, q12, #14 + vext.8 q14, q14, q14, #15 + veor d24, d24, d26 + bx lr +ENDPROC(__pmull16x64_p8) + + .macro pmull16x64_p64, v16, v64 + vmull.p64 q11, \v64\()l, \v16\()_L + vmull.p64 \v64, \v64\()h, \v16\()_H + veor \v64, \v64, q11 + .endm + // Fold reg1, reg2 into the next 32 data bytes, storing the result back // into reg1, reg2. - .macro fold_32_bytes, reg1, reg2 - vld1.64 {q11-q12}, [buf]! + .macro fold_32_bytes, reg1, reg2, p + vld1.64 {q8-q9}, [buf]! - vmull.p64 q8, \reg1\()h, FOLD_CONST_H - vmull.p64 \reg1, \reg1\()l, FOLD_CONST_L - vmull.p64 q9, \reg2\()h, FOLD_CONST_H - vmull.p64 \reg2, \reg2\()l, FOLD_CONST_L + pmull16x64_\p FOLD_CONST, \reg1 + pmull16x64_\p FOLD_CONST, \reg2 -CPU_LE( vrev64.8 q11, q11 ) -CPU_LE( vrev64.8 q12, q12 ) - vswp q11l, q11h - vswp q12l, q12h +CPU_LE( vrev64.8 q8, q8 ) +CPU_LE( vrev64.8 q9, q9 ) + vswp q8l, q8h + vswp q9l, q9h veor.8 \reg1, \reg1, q8 veor.8 \reg2, \reg2, q9 - veor.8 \reg1, \reg1, q11 - veor.8 \reg2, \reg2, q12 .endm // Fold src_reg into dst_reg, optionally loading the next fold constants - .macro fold_16_bytes, src_reg, dst_reg, load_next_consts - vmull.p64 q8, \src_reg\()l, FOLD_CONST_L - vmull.p64 \src_reg, \src_reg\()h, FOLD_CONST_H + .macro fold_16_bytes, src_reg, dst_reg, p, load_next_consts + pmull16x64_\p FOLD_CONST, \src_reg .ifnb \load_next_consts vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! .endif - veor.8 \dst_reg, \dst_reg, q8 veor.8 \dst_reg, \dst_reg, \src_reg .endm - .macro __adrl, out, sym - movw \out, #:lower16:\sym - movt \out, #:upper16:\sym - .endm - -// -// u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len); -// -// Assumes len >= 16. -// -ENTRY(crc_t10dif_pmull) - + .macro crct10dif, p // For sizes less than 256 bytes, we can't fold 128 bytes at a time. cmp len, #256 - blt .Lless_than_256_bytes + blt .Lless_than_256_bytes\@ - __adrl fold_consts_ptr, .Lfold_across_128_bytes_consts + mov_l fold_consts_ptr, .Lfold_across_128_bytes_consts // Load the first 128 data bytes. Byte swapping is necessary to make // the bit order match the polynomial coefficient order. @@ -199,27 +264,27 @@ CPU_LE( vrev64.8 q7, q7 ) // While >= 128 data bytes remain (not counting q0-q7), fold the 128 // bytes q0-q7 into them, storing the result back into q0-q7. -.Lfold_128_bytes_loop: - fold_32_bytes q0, q1 - fold_32_bytes q2, q3 - fold_32_bytes q4, q5 - fold_32_bytes q6, q7 +.Lfold_128_bytes_loop\@: + fold_32_bytes q0, q1, \p + fold_32_bytes q2, q3, \p + fold_32_bytes q4, q5, \p + fold_32_bytes q6, q7, \p subs len, len, #128 - bge .Lfold_128_bytes_loop + bge .Lfold_128_bytes_loop\@ // Now fold the 112 bytes in q0-q6 into the 16 bytes in q7. // Fold across 64 bytes. vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! - fold_16_bytes q0, q4 - fold_16_bytes q1, q5 - fold_16_bytes q2, q6 - fold_16_bytes q3, q7, 1 + fold_16_bytes q0, q4, \p + fold_16_bytes q1, q5, \p + fold_16_bytes q2, q6, \p + fold_16_bytes q3, q7, \p, 1 // Fold across 32 bytes. - fold_16_bytes q4, q6 - fold_16_bytes q5, q7, 1 + fold_16_bytes q4, q6, \p + fold_16_bytes q5, q7, \p, 1 // Fold across 16 bytes. - fold_16_bytes q6, q7 + fold_16_bytes q6, q7, \p // Add 128 to get the correct number of data bytes remaining in 0...127 // (not counting q7), following the previous extra subtraction by 128. @@ -229,25 +294,23 @@ CPU_LE( vrev64.8 q7, q7 ) // While >= 16 data bytes remain (not counting q7), fold the 16 bytes q7 // into them, storing the result back into q7. - blt .Lfold_16_bytes_loop_done -.Lfold_16_bytes_loop: - vmull.p64 q8, q7l, FOLD_CONST_L - vmull.p64 q7, q7h, FOLD_CONST_H - veor.8 q7, q7, q8 + blt .Lfold_16_bytes_loop_done\@ +.Lfold_16_bytes_loop\@: + pmull16x64_\p FOLD_CONST, q7 vld1.64 {q0}, [buf]! CPU_LE( vrev64.8 q0, q0 ) vswp q0l, q0h veor.8 q7, q7, q0 subs len, len, #16 - bge .Lfold_16_bytes_loop + bge .Lfold_16_bytes_loop\@ -.Lfold_16_bytes_loop_done: +.Lfold_16_bytes_loop_done\@: // Add 16 to get the correct number of data bytes remaining in 0...15 // (not counting q7), following the previous extra subtraction by 16. adds len, len, #16 - beq .Lreduce_final_16_bytes + beq .Lreduce_final_16_bytes\@ -.Lhandle_partial_segment: +.Lhandle_partial_segment\@: // Reduce the last '16 + len' bytes where 1 <= len <= 15 and the first // 16 bytes are in q7 and the rest are the remaining data in 'buf'. To // do this without needing a fold constant for each possible 'len', @@ -262,9 +325,9 @@ CPU_LE( vrev64.8 q0, q0 ) vswp q0l, q0h // q1 = high order part of second chunk: q7 left-shifted by 'len' bytes. - __adrl r3, .Lbyteshift_table + 16 - sub r3, r3, len - vld1.8 {q2}, [r3] + mov_l r1, .Lbyteshift_table + 16 + sub r1, r1, len + vld1.8 {q2}, [r1] vtbl.8 q1l, {q7l-q7h}, q2l vtbl.8 q1h, {q7l-q7h}, q2h @@ -282,12 +345,46 @@ CPU_LE( vrev64.8 q0, q0 ) vbsl.8 q2, q1, q0 // Fold the first chunk into the second chunk, storing the result in q7. - vmull.p64 q0, q3l, FOLD_CONST_L - vmull.p64 q7, q3h, FOLD_CONST_H - veor.8 q7, q7, q0 - veor.8 q7, q7, q2 + pmull16x64_\p FOLD_CONST, q3 + veor.8 q7, q3, q2 + b .Lreduce_final_16_bytes\@ + +.Lless_than_256_bytes\@: + // Checksumming a buffer of length 16...255 bytes + + mov_l fold_consts_ptr, .Lfold_across_16_bytes_consts + + // Load the first 16 data bytes. + vld1.64 {q7}, [buf]! +CPU_LE( vrev64.8 q7, q7 ) + vswp q7l, q7h + + // XOR the first 16 data *bits* with the initial CRC value. + vmov.i8 q0h, #0 + vmov.u16 q0h[3], init_crc + veor.8 q7h, q7h, q0h + + // Load the fold-across-16-bytes constants. + vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! + + cmp len, #16 + beq .Lreduce_final_16_bytes\@ // len == 16 + subs len, len, #32 + addlt len, len, #16 + blt .Lhandle_partial_segment\@ // 17 <= len <= 31 + b .Lfold_16_bytes_loop\@ // 32 <= len <= 255 + +.Lreduce_final_16_bytes\@: + .endm + +// +// u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len); +// +// Assumes len >= 16. +// +ENTRY(crc_t10dif_pmull64) + crct10dif p64 -.Lreduce_final_16_bytes: // Reduce the 128-bit value M(x), stored in q7, to the final 16-bit CRC. // Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'. @@ -320,32 +417,19 @@ CPU_LE( vrev64.8 q0, q0 ) vmov.u16 r0, q0l[0] bx lr +ENDPROC(crc_t10dif_pmull64) -.Lless_than_256_bytes: - // Checksumming a buffer of length 16...255 bytes +ENTRY(crc_t10dif_pmull8) + push {r4, lr} + mov_l r4, .L16x64perm - __adrl fold_consts_ptr, .Lfold_across_16_bytes_consts + crct10dif p8 - // Load the first 16 data bytes. - vld1.64 {q7}, [buf]! CPU_LE( vrev64.8 q7, q7 ) vswp q7l, q7h - - // XOR the first 16 data *bits* with the initial CRC value. - vmov.i8 q0h, #0 - vmov.u16 q0h[3], init_crc - veor.8 q7h, q7h, q0h - - // Load the fold-across-16-bytes constants. - vld1.64 {FOLD_CONSTS}, [fold_consts_ptr, :128]! - - cmp len, #16 - beq .Lreduce_final_16_bytes // len == 16 - subs len, len, #32 - addlt len, len, #16 - blt .Lhandle_partial_segment // 17 <= len <= 31 - b .Lfold_16_bytes_loop // 32 <= len <= 255 -ENDPROC(crc_t10dif_pmull) + vst1.64 {q7}, [r3, :128] + pop {r4, pc} +ENDPROC(crc_t10dif_pmull8) .section ".rodata", "a" .align 4 @@ -379,3 +463,6 @@ ENDPROC(crc_t10dif_pmull) .byte 0x88, 0x89, 0x8a, 0x8b, 0x8c, 0x8d, 0x8e, 0x8f .byte 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7 .byte 0x8, 0x9, 0xa, 0xb, 0xc, 0xd, 0xe , 0x0 + +.L16x64perm: + .quad 0x808080800000000, 0x909090901010101 diff --git a/arch/arm/crypto/crct10dif-ce-glue.c b/arch/arm/crypto/crct10dif-ce-glue.c index 79f3b204d8c0..a8b74523729e 100644 --- a/arch/arm/crypto/crct10dif-ce-glue.c +++ b/arch/arm/crypto/crct10dif-ce-glue.c @@ -19,7 +19,9 @@ #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U -asmlinkage u16 crc_t10dif_pmull(u16 init_crc, const u8 *buf, size_t len); +asmlinkage u16 crc_t10dif_pmull64(u16 init_crc, const u8 *buf, size_t len); +asmlinkage void crc_t10dif_pmull8(u16 init_crc, const u8 *buf, size_t len, + u8 out[16]); static int crct10dif_init(struct shash_desc *desc) { @@ -29,14 +31,14 @@ static int crct10dif_init(struct shash_desc *desc) return 0; } -static int crct10dif_update(struct shash_desc *desc, const u8 *data, - unsigned int length) +static int crct10dif_update_ce(struct shash_desc *desc, const u8 *data, + unsigned int length) { u16 *crc = shash_desc_ctx(desc); if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { kernel_neon_begin(); - *crc = crc_t10dif_pmull(*crc, data, length); + *crc = crc_t10dif_pmull64(*crc, data, length); kernel_neon_end(); } else { *crc = crc_t10dif_generic(*crc, data, length); @@ -45,6 +47,27 @@ static int crct10dif_update(struct shash_desc *desc, const u8 *data, return 0; } +static int crct10dif_update_neon(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + u16 *crcp = shash_desc_ctx(desc); + u8 buf[16] __aligned(16); + u16 crc = *crcp; + + if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { + kernel_neon_begin(); + crc_t10dif_pmull8(crc, data, length, buf); + kernel_neon_end(); + + crc = 0; + data = buf; + length = sizeof(buf); + } + + *crcp = crc_t10dif_generic(crc, data, length); + return 0; +} + static int crct10dif_final(struct shash_desc *desc, u8 *out) { u16 *crc = shash_desc_ctx(desc); @@ -53,10 +76,22 @@ static int crct10dif_final(struct shash_desc *desc, u8 *out) return 0; } -static struct shash_alg crc_t10dif_alg = { +static struct shash_alg algs[] = {{ + .digestsize = CRC_T10DIF_DIGEST_SIZE, + .init = crct10dif_init, + .update = crct10dif_update_neon, + .final = crct10dif_final, + .descsize = CRC_T10DIF_DIGEST_SIZE, + + .base.cra_name = "crct10dif", + .base.cra_driver_name = "crct10dif-arm-neon", + .base.cra_priority = 150, + .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, + .base.cra_module = THIS_MODULE, +}, { .digestsize = CRC_T10DIF_DIGEST_SIZE, .init = crct10dif_init, - .update = crct10dif_update, + .update = crct10dif_update_ce, .final = crct10dif_final, .descsize = CRC_T10DIF_DIGEST_SIZE, @@ -65,19 +100,19 @@ static struct shash_alg crc_t10dif_alg = { .base.cra_priority = 200, .base.cra_blocksize = CRC_T10DIF_BLOCK_SIZE, .base.cra_module = THIS_MODULE, -}; +}}; static int __init crc_t10dif_mod_init(void) { - if (!(elf_hwcap2 & HWCAP2_PMULL)) + if (!(elf_hwcap & HWCAP_NEON)) return -ENODEV; - return crypto_register_shash(&crc_t10dif_alg); + return crypto_register_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL)); } static void __exit crc_t10dif_mod_exit(void) { - crypto_unregister_shash(&crc_t10dif_alg); + crypto_unregister_shashes(algs, 1 + !!(elf_hwcap2 & HWCAP2_PMULL)); } module_init(crc_t10dif_mod_init); diff --git a/arch/arm64/crypto/crct10dif-ce-core.S b/arch/arm64/crypto/crct10dif-ce-core.S index 5604de61d06d..87dd6d46224d 100644 --- a/arch/arm64/crypto/crct10dif-ce-core.S +++ b/arch/arm64/crypto/crct10dif-ce-core.S @@ -1,8 +1,11 @@ // // Accelerated CRC-T10DIF using arm64 NEON and Crypto Extensions instructions // -// Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org> -// Copyright (C) 2019 Google LLC <ebiggers@google.com> +// Copyright (C) 2016 Linaro Ltd +// Copyright (C) 2019-2024 Google LLC +// +// Authors: Ard Biesheuvel <ardb@google.com> +// Eric Biggers <ebiggers@google.com> // // This program is free software; you can redistribute it and/or modify // it under the terms of the GNU General Public License version 2 as @@ -71,161 +74,117 @@ init_crc .req w0 buf .req x1 len .req x2 - fold_consts_ptr .req x3 + fold_consts_ptr .req x5 fold_consts .req v10 - ad .req v14 - - k00_16 .req v15 - k32_48 .req v16 - t3 .req v17 t4 .req v18 t5 .req v19 t6 .req v20 t7 .req v21 t8 .req v22 - t9 .req v23 - perm1 .req v24 - perm2 .req v25 - perm3 .req v26 - perm4 .req v27 - - bd1 .req v28 - bd2 .req v29 - bd3 .req v30 - bd4 .req v31 - - .macro __pmull_init_p64 - .endm + perm .req v27 - .macro __pmull_pre_p64, bd + .macro pmull16x64_p64, a16, b64, c64 + pmull2 \c64\().1q, \a16\().2d, \b64\().2d + pmull \b64\().1q, \a16\().1d, \b64\().1d .endm - .macro __pmull_init_p8 - // k00_16 := 0x0000000000000000_000000000000ffff - // k32_48 := 0x00000000ffffffff_0000ffffffffffff - movi k32_48.2d, #0xffffffff - mov k32_48.h[2], k32_48.h[0] - ushr k00_16.2d, k32_48.2d, #32 - - // prepare the permutation vectors - mov_q x5, 0x080f0e0d0c0b0a09 - movi perm4.8b, #8 - dup perm1.2d, x5 - eor perm1.16b, perm1.16b, perm4.16b - ushr perm2.2d, perm1.2d, #8 - ushr perm3.2d, perm1.2d, #16 - ushr perm4.2d, perm1.2d, #24 - sli perm2.2d, perm1.2d, #56 - sli perm3.2d, perm1.2d, #48 - sli perm4.2d, perm1.2d, #40 + /* + * Pairwise long polynomial multiplication of two 16-bit values + * + * { w0, w1 }, { y0, y1 } + * + * by two 64-bit values + * + * { x0, x1, x2, x3, x4, x5, x6, x7 }, { z0, z1, z2, z3, z4, z5, z6, z7 } + * + * where each vector element is a byte, ordered from least to most + * significant. + * + * This can be implemented using 8x8 long polynomial multiplication, by + * reorganizing the input so that each pairwise 8x8 multiplication + * produces one of the terms from the decomposition below, and + * combining the results of each rank and shifting them into place. + * + * Rank + * 0 w0*x0 ^ | y0*z0 ^ + * 1 (w0*x1 ^ w1*x0) << 8 ^ | (y0*z1 ^ y1*z0) << 8 ^ + * 2 (w0*x2 ^ w1*x1) << 16 ^ | (y0*z2 ^ y1*z1) << 16 ^ + * 3 (w0*x3 ^ w1*x2) << 24 ^ | (y0*z3 ^ y1*z2) << 24 ^ + * 4 (w0*x4 ^ w1*x3) << 32 ^ | (y0*z4 ^ y1*z3) << 32 ^ + * 5 (w0*x5 ^ w1*x4) << 40 ^ | (y0*z5 ^ y1*z4) << 40 ^ + * 6 (w0*x6 ^ w1*x5) << 48 ^ | (y0*z6 ^ y1*z5) << 48 ^ + * 7 (w0*x7 ^ w1*x6) << 56 ^ | (y0*z7 ^ y1*z6) << 56 ^ + * 8 w1*x7 << 64 | y1*z7 << 64 + * + * The inputs can be reorganized into + * + * { w0, w0, w0, w0, y0, y0, y0, y0 }, { w1, w1, w1, w1, y1, y1, y1, y1 } + * { x0, x2, x4, x6, z0, z2, z4, z6 }, { x1, x3, x5, x7, z1, z3, z5, z7 } + * + * and after performing 8x8->16 bit long polynomial multiplication of + * each of the halves of the first vector with those of the second one, + * we obtain the following four vectors of 16-bit elements: + * + * a := { w0*x0, w0*x2, w0*x4, w0*x6 }, { y0*z0, y0*z2, y0*z4, y0*z6 } + * b := { w0*x1, w0*x3, w0*x5, w0*x7 }, { y0*z1, y0*z3, y0*z5, y0*z7 } + * c := { w1*x0, w1*x2, w1*x4, w1*x6 }, { y1*z0, y1*z2, y1*z4, y1*z6 } + * d := { w1*x1, w1*x3, w1*x5, w1*x7 }, { y1*z1, y1*z3, y1*z5, y1*z7 } + * + * Results b and c can be XORed together, as the vector elements have + * matching ranks. Then, the final XOR (*) can be pulled forward, and + * applied between the halves of each of the remaining three vectors, + * which are then shifted into place, and combined to produce two + * 80-bit results. + * + * (*) NOTE: the 16x64 bit polynomial multiply below is not equivalent + * to the 64x64 bit one above, but XOR'ing the outputs together will + * produce the expected result, and this is sufficient in the context of + * this algorithm. + */ + .macro pmull16x64_p8, a16, b64, c64 + ext t7.16b, \b64\().16b, \b64\().16b, #1 + tbl t5.16b, {\a16\().16b}, perm.16b + uzp1 t7.16b, \b64\().16b, t7.16b + bl __pmull_p8_16x64 + ext \b64\().16b, t4.16b, t4.16b, #15 + eor \c64\().16b, t8.16b, t5.16b .endm - .macro __pmull_pre_p8, bd - tbl bd1.16b, {\bd\().16b}, perm1.16b - tbl bd2.16b, {\bd\().16b}, perm2.16b - tbl bd3.16b, {\bd\().16b}, perm3.16b - tbl bd4.16b, {\bd\().16b}, perm4.16b - .endm - -SYM_FUNC_START_LOCAL(__pmull_p8_core) -.L__pmull_p8_core: - ext t4.8b, ad.8b, ad.8b, #1 // A1 - ext t5.8b, ad.8b, ad.8b, #2 // A2 - ext t6.8b, ad.8b, ad.8b, #3 // A3 - - pmull t4.8h, t4.8b, fold_consts.8b // F = A1*B - pmull t8.8h, ad.8b, bd1.8b // E = A*B1 - pmull t5.8h, t5.8b, fold_consts.8b // H = A2*B - pmull t7.8h, ad.8b, bd2.8b // G = A*B2 - pmull t6.8h, t6.8b, fold_consts.8b // J = A3*B - pmull t9.8h, ad.8b, bd3.8b // I = A*B3 - pmull t3.8h, ad.8b, bd4.8b // K = A*B4 - b 0f - -.L__pmull_p8_core2: - tbl t4.16b, {ad.16b}, perm1.16b // A1 - tbl t5.16b, {ad.16b}, perm2.16b // A2 - tbl t6.16b, {ad.16b}, perm3.16b // A3 - - pmull2 t4.8h, t4.16b, fold_consts.16b // F = A1*B - pmull2 t8.8h, ad.16b, bd1.16b // E = A*B1 - pmull2 t5.8h, t5.16b, fold_consts.16b // H = A2*B - pmull2 t7.8h, ad.16b, bd2.16b // G = A*B2 - pmull2 t6.8h, t6.16b, fold_consts.16b // J = A3*B - pmull2 t9.8h, ad.16b, bd3.16b // I = A*B3 - pmull2 t3.8h, ad.16b, bd4.16b // K = A*B4 - -0: eor t4.16b, t4.16b, t8.16b // L = E + F - eor t5.16b, t5.16b, t7.16b // M = G + H - eor t6.16b, t6.16b, t9.16b // N = I + J - - uzp1 t8.2d, t4.2d, t5.2d - uzp2 t4.2d, t4.2d, t5.2d - uzp1 t7.2d, t6.2d, t3.2d - uzp2 t6.2d, t6.2d, t3.2d - - // t4 = (L) (P0 + P1) << 8 - // t5 = (M) (P2 + P3) << 16 - eor t8.16b, t8.16b, t4.16b - and t4.16b, t4.16b, k32_48.16b - - // t6 = (N) (P4 + P5) << 24 - // t7 = (K) (P6 + P7) << 32 - eor t7.16b, t7.16b, t6.16b - and t6.16b, t6.16b, k00_16.16b - - eor t8.16b, t8.16b, t4.16b - eor t7.16b, t7.16b, t6.16b - - zip2 t5.2d, t8.2d, t4.2d - zip1 t4.2d, t8.2d, t4.2d - zip2 t3.2d, t7.2d, t6.2d - zip1 t6.2d, t7.2d, t6.2d - - ext t4.16b, t4.16b, t4.16b, #15 +SYM_FUNC_START_LOCAL(__pmull_p8_16x64) + ext t6.16b, t5.16b, t5.16b, #8 + + pmull t3.8h, t7.8b, t5.8b + pmull t4.8h, t7.8b, t6.8b + pmull2 t5.8h, t7.16b, t5.16b + pmull2 t6.8h, t7.16b, t6.16b + + ext t8.16b, t3.16b, t3.16b, #8 + eor t4.16b, t4.16b, t6.16b + ext t7.16b, t5.16b, t5.16b, #8 + ext t6.16b, t4.16b, t4.16b, #8 + eor t8.8b, t8.8b, t3.8b + eor t5.8b, t5.8b, t7.8b + eor t4.8b, t4.8b, t6.8b ext t5.16b, t5.16b, t5.16b, #14 - ext t6.16b, t6.16b, t6.16b, #13 - ext t3.16b, t3.16b, t3.16b, #12 - - eor t4.16b, t4.16b, t5.16b - eor t6.16b, t6.16b, t3.16b ret -SYM_FUNC_END(__pmull_p8_core) +SYM_FUNC_END(__pmull_p8_16x64) - .macro __pmull_p8, rq, ad, bd, i - .ifnc \bd, fold_consts - .err - .endif - mov ad.16b, \ad\().16b - .ifb \i - pmull \rq\().8h, \ad\().8b, \bd\().8b // D = A*B - .else - pmull2 \rq\().8h, \ad\().16b, \bd\().16b // D = A*B - .endif - - bl .L__pmull_p8_core\i - - eor \rq\().16b, \rq\().16b, t4.16b - eor \rq\().16b, \rq\().16b, t6.16b - .endm // Fold reg1, reg2 into the next 32 data bytes, storing the result back // into reg1, reg2. .macro fold_32_bytes, p, reg1, reg2 ldp q11, q12, [buf], #0x20 - __pmull_\p v8, \reg1, fold_consts, 2 - __pmull_\p \reg1, \reg1, fold_consts + pmull16x64_\p fold_consts, \reg1, v8 CPU_LE( rev64 v11.16b, v11.16b ) CPU_LE( rev64 v12.16b, v12.16b ) - __pmull_\p v9, \reg2, fold_consts, 2 - __pmull_\p \reg2, \reg2, fold_consts + pmull16x64_\p fold_consts, \reg2, v9 CPU_LE( ext v11.16b, v11.16b, v11.16b, #8 ) CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) @@ -238,26 +197,15 @@ CPU_LE( ext v12.16b, v12.16b, v12.16b, #8 ) // Fold src_reg into dst_reg, optionally loading the next fold constants .macro fold_16_bytes, p, src_reg, dst_reg, load_next_consts - __pmull_\p v8, \src_reg, fold_consts - __pmull_\p \src_reg, \src_reg, fold_consts, 2 + pmull16x64_\p fold_consts, \src_reg, v8 .ifnb \load_next_consts ld1 {fold_consts.2d}, [fold_consts_ptr], #16 - __pmull_pre_\p fold_consts .endif eor \dst_reg\().16b, \dst_reg\().16b, v8.16b eor \dst_reg\().16b, \dst_reg\().16b, \src_reg\().16b .endm - .macro __pmull_p64, rd, rn, rm, n - .ifb \n - pmull \rd\().1q, \rn\().1d, \rm\().1d - .else - pmull2 \rd\().1q, \rn\().2d, \rm\().2d - .endif - .endm - .macro crc_t10dif_pmull, p - __pmull_init_\p // For sizes less than 256 bytes, we can't fold 128 bytes at a time. cmp len, #256 @@ -296,7 +244,6 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // Load the constants for folding across 128 bytes. ld1 {fold_consts.2d}, [fold_consts_ptr] - __pmull_pre_\p fold_consts // Subtract 128 for the 128 data bytes just consumed. Subtract another // 128 to simplify the termination condition of the following loop. @@ -318,7 +265,6 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // Fold across 64 bytes. add fold_consts_ptr, fold_consts_ptr, #16 ld1 {fold_consts.2d}, [fold_consts_ptr], #16 - __pmull_pre_\p fold_consts fold_16_bytes \p, v0, v4 fold_16_bytes \p, v1, v5 fold_16_bytes \p, v2, v6 @@ -339,8 +285,7 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // into them, storing the result back into v7. b.lt .Lfold_16_bytes_loop_done_\@ .Lfold_16_bytes_loop_\@: - __pmull_\p v8, v7, fold_consts - __pmull_\p v7, v7, fold_consts, 2 + pmull16x64_\p fold_consts, v7, v8 eor v7.16b, v7.16b, v8.16b ldr q0, [buf], #16 CPU_LE( rev64 v0.16b, v0.16b ) @@ -387,51 +332,10 @@ CPU_LE( ext v0.16b, v0.16b, v0.16b, #8 ) bsl v2.16b, v1.16b, v0.16b // Fold the first chunk into the second chunk, storing the result in v7. - __pmull_\p v0, v3, fold_consts - __pmull_\p v7, v3, fold_consts, 2 - eor v7.16b, v7.16b, v0.16b + pmull16x64_\p fold_consts, v3, v0 + eor v7.16b, v3.16b, v0.16b eor v7.16b, v7.16b, v2.16b - -.Lreduce_final_16_bytes_\@: - // Reduce the 128-bit value M(x), stored in v7, to the final 16-bit CRC. - - movi v2.16b, #0 // init zero register - - // Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'. - ld1 {fold_consts.2d}, [fold_consts_ptr], #16 - __pmull_pre_\p fold_consts - - // Fold the high 64 bits into the low 64 bits, while also multiplying by - // x^64. This produces a 128-bit value congruent to x^64 * M(x) and - // whose low 48 bits are 0. - ext v0.16b, v2.16b, v7.16b, #8 - __pmull_\p v7, v7, fold_consts, 2 // high bits * x^48 * (x^80 mod G(x)) - eor v0.16b, v0.16b, v7.16b // + low bits * x^64 - - // Fold the high 32 bits into the low 96 bits. This produces a 96-bit - // value congruent to x^64 * M(x) and whose low 48 bits are 0. - ext v1.16b, v0.16b, v2.16b, #12 // extract high 32 bits - mov v0.s[3], v2.s[0] // zero high 32 bits - __pmull_\p v1, v1, fold_consts // high 32 bits * x^48 * (x^48 mod G(x)) - eor v0.16b, v0.16b, v1.16b // + low bits - - // Load G(x) and floor(x^48 / G(x)). - ld1 {fold_consts.2d}, [fold_consts_ptr] - __pmull_pre_\p fold_consts - - // Use Barrett reduction to compute the final CRC value. - __pmull_\p v1, v0, fold_consts, 2 // high 32 bits * floor(x^48 / G(x)) - ushr v1.2d, v1.2d, #32 // /= x^32 - __pmull_\p v1, v1, fold_consts // *= G(x) - ushr v0.2d, v0.2d, #48 - eor v0.16b, v0.16b, v1.16b // + low 16 nonzero bits - // Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of v0. - - umov w0, v0.h[0] - .ifc \p, p8 - frame_pop - .endif - ret + b .Lreduce_final_16_bytes_\@ .Lless_than_256_bytes_\@: // Checksumming a buffer of length 16...255 bytes @@ -450,7 +354,6 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // Load the fold-across-16-bytes constants. ld1 {fold_consts.2d}, [fold_consts_ptr], #16 - __pmull_pre_\p fold_consts cmp len, #16 b.eq .Lreduce_final_16_bytes_\@ // len == 16 @@ -458,6 +361,8 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) b.ge .Lfold_16_bytes_loop_\@ // 32 <= len <= 255 add len, len, #16 b .Lhandle_partial_segment_\@ // 17 <= len <= 31 + +.Lreduce_final_16_bytes_\@: .endm // @@ -467,7 +372,22 @@ CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) // SYM_FUNC_START(crc_t10dif_pmull_p8) frame_push 1 + + // Compose { 0,0,0,0, 8,8,8,8, 1,1,1,1, 9,9,9,9 } + movi perm.4h, #8, lsl #8 + orr perm.2s, #1, lsl #16 + orr perm.2s, #1, lsl #24 + zip1 perm.16b, perm.16b, perm.16b + zip1 perm.16b, perm.16b, perm.16b + crc_t10dif_pmull p8 + +CPU_LE( rev64 v7.16b, v7.16b ) +CPU_LE( ext v7.16b, v7.16b, v7.16b, #8 ) + str q7, [x3] + + frame_pop + ret SYM_FUNC_END(crc_t10dif_pmull_p8) .align 5 @@ -478,6 +398,41 @@ SYM_FUNC_END(crc_t10dif_pmull_p8) // SYM_FUNC_START(crc_t10dif_pmull_p64) crc_t10dif_pmull p64 + + // Reduce the 128-bit value M(x), stored in v7, to the final 16-bit CRC. + + movi v2.16b, #0 // init zero register + + // Load 'x^48 * (x^48 mod G(x))' and 'x^48 * (x^80 mod G(x))'. + ld1 {fold_consts.2d}, [fold_consts_ptr], #16 + + // Fold the high 64 bits into the low 64 bits, while also multiplying by + // x^64. This produces a 128-bit value congruent to x^64 * M(x) and + // whose low 48 bits are 0. + ext v0.16b, v2.16b, v7.16b, #8 + pmull2 v7.1q, v7.2d, fold_consts.2d // high bits * x^48 * (x^80 mod G(x)) + eor v0.16b, v0.16b, v7.16b // + low bits * x^64 + + // Fold the high 32 bits into the low 96 bits. This produces a 96-bit + // value congruent to x^64 * M(x) and whose low 48 bits are 0. + ext v1.16b, v0.16b, v2.16b, #12 // extract high 32 bits + mov v0.s[3], v2.s[0] // zero high 32 bits + pmull v1.1q, v1.1d, fold_consts.1d // high 32 bits * x^48 * (x^48 mod G(x)) + eor v0.16b, v0.16b, v1.16b // + low bits + + // Load G(x) and floor(x^48 / G(x)). + ld1 {fold_consts.2d}, [fold_consts_ptr] + + // Use Barrett reduction to compute the final CRC value. + pmull2 v1.1q, v0.2d, fold_consts.2d // high 32 bits * floor(x^48 / G(x)) + ushr v1.2d, v1.2d, #32 // /= x^32 + pmull v1.1q, v1.1d, fold_consts.1d // *= G(x) + ushr v0.2d, v0.2d, #48 + eor v0.16b, v0.16b, v1.16b // + low 16 nonzero bits + // Final CRC value (x^16 * M(x)) mod G(x) is in low 16 bits of v0. + + umov w0, v0.h[0] + ret SYM_FUNC_END(crc_t10dif_pmull_p64) .section ".rodata", "a" diff --git a/arch/arm64/crypto/crct10dif-ce-glue.c b/arch/arm64/crypto/crct10dif-ce-glue.c index 606d25c559ed..08bcbd884395 100644 --- a/arch/arm64/crypto/crct10dif-ce-glue.c +++ b/arch/arm64/crypto/crct10dif-ce-glue.c @@ -20,7 +20,8 @@ #define CRC_T10DIF_PMULL_CHUNK_SIZE 16U -asmlinkage u16 crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len); +asmlinkage void crc_t10dif_pmull_p8(u16 init_crc, const u8 *buf, size_t len, + u8 out[16]); asmlinkage u16 crc_t10dif_pmull_p64(u16 init_crc, const u8 *buf, size_t len); static int crct10dif_init(struct shash_desc *desc) @@ -34,25 +35,21 @@ static int crct10dif_init(struct shash_desc *desc) static int crct10dif_update_pmull_p8(struct shash_desc *desc, const u8 *data, unsigned int length) { - u16 *crc = shash_desc_ctx(desc); - - if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - do { - unsigned int chunk = length; - - if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) - chunk = SZ_4K; - - kernel_neon_begin(); - *crc = crc_t10dif_pmull_p8(*crc, data, chunk); - kernel_neon_end(); - data += chunk; - length -= chunk; - } while (length); - } else { - *crc = crc_t10dif_generic(*crc, data, length); + u16 *crcp = shash_desc_ctx(desc); + u16 crc = *crcp; + u8 buf[16]; + + if (length > CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { + kernel_neon_begin(); + crc_t10dif_pmull_p8(crc, data, length, buf); + kernel_neon_end(); + + crc = 0; + data = buf; + length = sizeof(buf); } + *crcp = crc_t10dif_generic(crc, data, length); return 0; } @@ -62,18 +59,9 @@ static int crct10dif_update_pmull_p64(struct shash_desc *desc, const u8 *data, u16 *crc = shash_desc_ctx(desc); if (length >= CRC_T10DIF_PMULL_CHUNK_SIZE && crypto_simd_usable()) { - do { - unsigned int chunk = length; - - if (chunk > SZ_4K + CRC_T10DIF_PMULL_CHUNK_SIZE) - chunk = SZ_4K; - - kernel_neon_begin(); - *crc = crc_t10dif_pmull_p64(*crc, data, chunk); - kernel_neon_end(); - data += chunk; - length -= chunk; - } while (length); + kernel_neon_begin(); + *crc = crc_t10dif_pmull_p64(*crc, data, length); + kernel_neon_end(); } else { *crc = crc_t10dif_generic(*crc, data, length); } diff --git a/arch/powerpc/crypto/Kconfig b/arch/powerpc/crypto/Kconfig index 46a4c85e85e2..951a43726461 100644 --- a/arch/powerpc/crypto/Kconfig +++ b/arch/powerpc/crypto/Kconfig @@ -107,12 +107,12 @@ config CRYPTO_AES_PPC_SPE config CRYPTO_AES_GCM_P10 tristate "Stitched AES/GCM acceleration support on P10 or later CPU (PPC)" - depends on BROKEN depends on PPC64 && CPU_LITTLE_ENDIAN && VSX select CRYPTO_LIB_AES select CRYPTO_ALGAPI select CRYPTO_AEAD select CRYPTO_SKCIPHER + select CRYPTO_SIMD help AEAD cipher: AES cipher algorithms (FIPS-197) GCM (Galois/Counter Mode) authenticated encryption mode (NIST SP800-38D) diff --git a/arch/powerpc/crypto/aes-gcm-p10-glue.c b/arch/powerpc/crypto/aes-gcm-p10-glue.c index f66ad56e765f..f37b3d13fc53 100644 --- a/arch/powerpc/crypto/aes-gcm-p10-glue.c +++ b/arch/powerpc/crypto/aes-gcm-p10-glue.c @@ -8,6 +8,7 @@ #include <linux/unaligned.h> #include <asm/simd.h> #include <asm/switch_to.h> +#include <crypto/gcm.h> #include <crypto/aes.h> #include <crypto/algapi.h> #include <crypto/b128ops.h> @@ -24,6 +25,7 @@ #define PPC_ALIGN 16 #define GCM_IV_SIZE 12 +#define RFC4106_NONCE_SIZE 4 MODULE_DESCRIPTION("PPC64le AES-GCM with Stitched implementation"); MODULE_AUTHOR("Danny Tsen <dtsen@linux.ibm.com"); @@ -31,7 +33,7 @@ MODULE_LICENSE("GPL v2"); MODULE_ALIAS_CRYPTO("aes"); asmlinkage int aes_p10_set_encrypt_key(const u8 *userKey, const int bits, - void *key); + void *key); asmlinkage void aes_p10_encrypt(const u8 *in, u8 *out, const void *key); asmlinkage void aes_p10_gcm_encrypt(u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); @@ -39,7 +41,8 @@ asmlinkage void aes_p10_gcm_decrypt(u8 *in, u8 *out, size_t len, void *rkey, u8 *iv, void *Xi); asmlinkage void gcm_init_htable(unsigned char htable[], unsigned char Xi[]); asmlinkage void gcm_ghash_p10(unsigned char *Xi, unsigned char *Htable, - unsigned char *aad, unsigned int alen); + unsigned char *aad, unsigned int alen); +asmlinkage void gcm_update(u8 *iv, void *Xi); struct aes_key { u8 key[AES_MAX_KEYLENGTH]; @@ -52,6 +55,7 @@ struct gcm_ctx { u8 aad_hash[16]; u64 aadLen; u64 Plen; /* offset 56 - used in aes_p10_gcm_{en/de}crypt */ + u8 pblock[16]; }; struct Hash_ctx { u8 H[16]; /* subkey */ @@ -60,17 +64,20 @@ struct Hash_ctx { struct p10_aes_gcm_ctx { struct aes_key enc_key; + u8 nonce[RFC4106_NONCE_SIZE]; }; static void vsx_begin(void) { preempt_disable(); + pagefault_disable(); enable_kernel_vsx(); } static void vsx_end(void) { disable_kernel_vsx(); + pagefault_enable(); preempt_enable(); } @@ -185,7 +192,7 @@ static int set_authsize(struct crypto_aead *tfm, unsigned int authsize) } static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, - unsigned int keylen) + unsigned int keylen) { struct crypto_tfm *tfm = crypto_aead_tfm(aead); struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); @@ -198,7 +205,8 @@ static int p10_aes_gcm_setkey(struct crypto_aead *aead, const u8 *key, return ret ? -EINVAL : 0; } -static int p10_aes_gcm_crypt(struct aead_request *req, int enc) +static int p10_aes_gcm_crypt(struct aead_request *req, u8 *riv, + int assoclen, int enc) { struct crypto_tfm *tfm = req->base.tfm; struct p10_aes_gcm_ctx *ctx = crypto_tfm_ctx(tfm); @@ -210,7 +218,6 @@ static int p10_aes_gcm_crypt(struct aead_request *req, int enc) struct skcipher_walk walk; u8 *assocmem = NULL; u8 *assoc; - unsigned int assoclen = req->assoclen; unsigned int cryptlen = req->cryptlen; unsigned char ivbuf[AES_BLOCK_SIZE+PPC_ALIGN]; unsigned char *iv = PTR_ALIGN((void *)ivbuf, PPC_ALIGN); @@ -218,11 +225,12 @@ static int p10_aes_gcm_crypt(struct aead_request *req, int enc) unsigned long auth_tag_len = crypto_aead_authsize(__crypto_aead_cast(tfm)); u8 otag[16]; int total_processed = 0; + int nbytes; memset(databuf, 0, sizeof(databuf)); memset(hashbuf, 0, sizeof(hashbuf)); memset(ivbuf, 0, sizeof(ivbuf)); - memcpy(iv, req->iv, GCM_IV_SIZE); + memcpy(iv, riv, GCM_IV_SIZE); /* Linearize assoc, if not already linear */ if (req->src->length >= assoclen && req->src->length) { @@ -257,19 +265,25 @@ static int p10_aes_gcm_crypt(struct aead_request *req, int enc) if (ret) return ret; - while (walk.nbytes > 0 && ret == 0) { + while ((nbytes = walk.nbytes) > 0 && ret == 0) { + u8 *src = walk.src.virt.addr; + u8 *dst = walk.dst.virt.addr; + u8 buf[AES_BLOCK_SIZE]; + + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) + src = dst = memcpy(buf, src, nbytes); vsx_begin(); if (enc) - aes_p10_gcm_encrypt(walk.src.virt.addr, - walk.dst.virt.addr, - walk.nbytes, + aes_p10_gcm_encrypt(src, dst, nbytes, &ctx->enc_key, gctx->iv, hash->Htable); else - aes_p10_gcm_decrypt(walk.src.virt.addr, - walk.dst.virt.addr, - walk.nbytes, + aes_p10_gcm_decrypt(src, dst, nbytes, &ctx->enc_key, gctx->iv, hash->Htable); + + if (unlikely(nbytes > 0 && nbytes < AES_BLOCK_SIZE)) + memcpy(walk.dst.virt.addr, buf, nbytes); + vsx_end(); total_processed += walk.nbytes; @@ -281,6 +295,7 @@ static int p10_aes_gcm_crypt(struct aead_request *req, int enc) /* Finalize hash */ vsx_begin(); + gcm_update(gctx->iv, hash->Htable); finish_tag(gctx, hash, total_processed); vsx_end(); @@ -302,17 +317,63 @@ static int p10_aes_gcm_crypt(struct aead_request *req, int enc) return 0; } +static int rfc4106_setkey(struct crypto_aead *tfm, const u8 *inkey, + unsigned int keylen) +{ + struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(tfm); + int err; + + keylen -= RFC4106_NONCE_SIZE; + err = p10_aes_gcm_setkey(tfm, inkey, keylen); + if (err) + return err; + + memcpy(ctx->nonce, inkey + keylen, RFC4106_NONCE_SIZE); + return 0; +} + +static int rfc4106_setauthsize(struct crypto_aead *tfm, unsigned int authsize) +{ + return crypto_rfc4106_check_authsize(authsize); +} + +static int rfc4106_encrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); + u8 iv[AES_BLOCK_SIZE]; + + memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); + memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); + + return crypto_ipsec_check_assoclen(req->assoclen) ?: + p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 1); +} + +static int rfc4106_decrypt(struct aead_request *req) +{ + struct crypto_aead *aead = crypto_aead_reqtfm(req); + struct p10_aes_gcm_ctx *ctx = crypto_aead_ctx(aead); + u8 iv[AES_BLOCK_SIZE]; + + memcpy(iv, ctx->nonce, RFC4106_NONCE_SIZE); + memcpy(iv + RFC4106_NONCE_SIZE, req->iv, GCM_RFC4106_IV_SIZE); + + return crypto_ipsec_check_assoclen(req->assoclen) ?: + p10_aes_gcm_crypt(req, iv, req->assoclen - GCM_RFC4106_IV_SIZE, 0); +} + static int p10_aes_gcm_encrypt(struct aead_request *req) { - return p10_aes_gcm_crypt(req, 1); + return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 1); } static int p10_aes_gcm_decrypt(struct aead_request *req) { - return p10_aes_gcm_crypt(req, 0); + return p10_aes_gcm_crypt(req, req->iv, req->assoclen, 0); } -static struct aead_alg gcm_aes_alg = { +static struct aead_alg gcm_aes_algs[] = {{ .ivsize = GCM_IV_SIZE, .maxauthsize = 16, @@ -321,23 +382,57 @@ static struct aead_alg gcm_aes_alg = { .encrypt = p10_aes_gcm_encrypt, .decrypt = p10_aes_gcm_decrypt, - .base.cra_name = "gcm(aes)", - .base.cra_driver_name = "aes_gcm_p10", + .base.cra_name = "__gcm(aes)", + .base.cra_driver_name = "__aes_gcm_p10", .base.cra_priority = 2100, .base.cra_blocksize = 1, - .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx), + .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx)+ + 4 * sizeof(u64[2]), .base.cra_module = THIS_MODULE, -}; + .base.cra_flags = CRYPTO_ALG_INTERNAL, +}, { + .ivsize = GCM_RFC4106_IV_SIZE, + .maxauthsize = 16, + .setkey = rfc4106_setkey, + .setauthsize = rfc4106_setauthsize, + .encrypt = rfc4106_encrypt, + .decrypt = rfc4106_decrypt, + + .base.cra_name = "__rfc4106(gcm(aes))", + .base.cra_driver_name = "__rfc4106_aes_gcm_p10", + .base.cra_priority = 2100, + .base.cra_blocksize = 1, + .base.cra_ctxsize = sizeof(struct p10_aes_gcm_ctx) + + 4 * sizeof(u64[2]), + .base.cra_module = THIS_MODULE, + .base.cra_flags = CRYPTO_ALG_INTERNAL, +}}; + +static struct simd_aead_alg *p10_simd_aeads[ARRAY_SIZE(gcm_aes_algs)]; static int __init p10_init(void) { - return crypto_register_aead(&gcm_aes_alg); + int ret; + + if (!cpu_has_feature(CPU_FTR_ARCH_31)) + return 0; + + ret = simd_register_aeads_compat(gcm_aes_algs, + ARRAY_SIZE(gcm_aes_algs), + p10_simd_aeads); + if (ret) { + simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), + p10_simd_aeads); + return ret; + } + return 0; } static void __exit p10_exit(void) { - crypto_unregister_aead(&gcm_aes_alg); + simd_unregister_aeads(gcm_aes_algs, ARRAY_SIZE(gcm_aes_algs), + p10_simd_aeads); } -module_cpu_feature_match(PPC_MODULE_FEATURE_P10, p10_init); +module_init(p10_init); module_exit(p10_exit); diff --git a/arch/powerpc/crypto/aes-gcm-p10.S b/arch/powerpc/crypto/aes-gcm-p10.S index a51f4b265308..89f50eef3512 100644 --- a/arch/powerpc/crypto/aes-gcm-p10.S +++ b/arch/powerpc/crypto/aes-gcm-p10.S @@ -1,42 +1,42 @@ /* SPDX-License-Identifier: GPL-2.0-or-later */ - # - # Accelerated AES-GCM stitched implementation for ppc64le. - # - # Copyright 2022- IBM Inc. All rights reserved - # - #=================================================================================== - # Written by Danny Tsen <dtsen@linux.ibm.com> - # - # GHASH is based on the Karatsuba multiplication method. - # - # Xi xor X1 - # - # X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H = - # (X1.h * H4.h + xX.l * H4.l + X1 * H4) + - # (X2.h * H3.h + X2.l * H3.l + X2 * H3) + - # (X3.h * H2.h + X3.l * H2.l + X3 * H2) + - # (X4.h * H.h + X4.l * H.l + X4 * H) - # - # Xi = v0 - # H Poly = v2 - # Hash keys = v3 - v14 - # ( H.l, H, H.h) - # ( H^2.l, H^2, H^2.h) - # ( H^3.l, H^3, H^3.h) - # ( H^4.l, H^4, H^4.h) - # - # v30 is IV - # v31 - counter 1 - # - # AES used, - # vs0 - vs14 for round keys - # v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted) - # - # This implementation uses stitched AES-GCM approach to improve overall performance. - # AES is implemented with 8x blocks and GHASH is using 2 4x blocks. - # - # =================================================================================== - # +# +# Accelerated AES-GCM stitched implementation for ppc64le. +# +# Copyright 2024- IBM Inc. +# +#=================================================================================== +# Written by Danny Tsen <dtsen@us.ibm.com> +# +# GHASH is based on the Karatsuba multiplication method. +# +# Xi xor X1 +# +# X1 * H^4 + X2 * H^3 + x3 * H^2 + X4 * H = +# (X1.h * H4.h + xX.l * H4.l + X1 * H4) + +# (X2.h * H3.h + X2.l * H3.l + X2 * H3) + +# (X3.h * H2.h + X3.l * H2.l + X3 * H2) + +# (X4.h * H.h + X4.l * H.l + X4 * H) +# +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# ( H.l, H, H.h) +# ( H^2.l, H^2, H^2.h) +# ( H^3.l, H^3, H^3.h) +# ( H^4.l, H^4, H^4.h) +# +# v30 is IV +# v31 - counter 1 +# +# AES used, +# vs0 - round key 0 +# v15, v16, v17, v18, v19, v20, v21, v22 for 8 blocks (encrypted) +# +# This implementation uses stitched AES-GCM approach to improve overall performance. +# AES is implemented with 8x blocks and GHASH is using 2 4x blocks. +# +# =================================================================================== +# #include <asm/ppc_asm.h> #include <linux/linkage.h> @@ -44,483 +44,224 @@ .machine "any" .text - # 4x loops - # v15 - v18 - input states - # vs1 - vs9 - round keys - # -.macro Loop_aes_middle4x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - vcipher 15, 15, 19 - vcipher 16, 16, 19 - vcipher 17, 17, 19 - vcipher 18, 18, 19 - - vcipher 15, 15, 20 - vcipher 16, 16, 20 - vcipher 17, 17, 20 - vcipher 18, 18, 20 - - vcipher 15, 15, 21 - vcipher 16, 16, 21 - vcipher 17, 17, 21 - vcipher 18, 18, 21 - - vcipher 15, 15, 22 - vcipher 16, 16, 22 - vcipher 17, 17, 22 - vcipher 18, 18, 22 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - vcipher 15, 15, 19 - vcipher 16, 16, 19 - vcipher 17, 17, 19 - vcipher 18, 18, 19 - - vcipher 15, 15, 20 - vcipher 16, 16, 20 - vcipher 17, 17, 20 - vcipher 18, 18, 20 - - vcipher 15, 15, 21 - vcipher 16, 16, 21 - vcipher 17, 17, 21 - vcipher 18, 18, 21 - - vcipher 15, 15, 22 - vcipher 16, 16, 22 - vcipher 17, 17, 22 - vcipher 18, 18, 22 - - xxlor 23+32, 9, 9 - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 +.macro SAVE_GPR GPR OFFSET FRAME + std \GPR,\OFFSET(\FRAME) .endm - # 8x loops - # v15 - v22 - input states - # vs1 - vs9 - round keys - # -.macro Loop_aes_middle8x - xxlor 23+32, 1, 1 - xxlor 24+32, 2, 2 - xxlor 25+32, 3, 3 - xxlor 26+32, 4, 4 - - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 - vcipher 19, 19, 23 - vcipher 20, 20, 23 - vcipher 21, 21, 23 - vcipher 22, 22, 23 - - vcipher 15, 15, 24 - vcipher 16, 16, 24 - vcipher 17, 17, 24 - vcipher 18, 18, 24 - vcipher 19, 19, 24 - vcipher 20, 20, 24 - vcipher 21, 21, 24 - vcipher 22, 22, 24 - - vcipher 15, 15, 25 - vcipher 16, 16, 25 - vcipher 17, 17, 25 - vcipher 18, 18, 25 - vcipher 19, 19, 25 - vcipher 20, 20, 25 - vcipher 21, 21, 25 - vcipher 22, 22, 25 - - vcipher 15, 15, 26 - vcipher 16, 16, 26 - vcipher 17, 17, 26 - vcipher 18, 18, 26 - vcipher 19, 19, 26 - vcipher 20, 20, 26 - vcipher 21, 21, 26 - vcipher 22, 22, 26 - - xxlor 23+32, 5, 5 - xxlor 24+32, 6, 6 - xxlor 25+32, 7, 7 - xxlor 26+32, 8, 8 - - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 - vcipher 19, 19, 23 - vcipher 20, 20, 23 - vcipher 21, 21, 23 - vcipher 22, 22, 23 - - vcipher 15, 15, 24 - vcipher 16, 16, 24 - vcipher 17, 17, 24 - vcipher 18, 18, 24 - vcipher 19, 19, 24 - vcipher 20, 20, 24 - vcipher 21, 21, 24 - vcipher 22, 22, 24 - - vcipher 15, 15, 25 - vcipher 16, 16, 25 - vcipher 17, 17, 25 - vcipher 18, 18, 25 - vcipher 19, 19, 25 - vcipher 20, 20, 25 - vcipher 21, 21, 25 - vcipher 22, 22, 25 - - vcipher 15, 15, 26 - vcipher 16, 16, 26 - vcipher 17, 17, 26 - vcipher 18, 18, 26 - vcipher 19, 19, 26 - vcipher 20, 20, 26 - vcipher 21, 21, 26 - vcipher 22, 22, 26 - - xxlor 23+32, 9, 9 - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 - vcipher 19, 19, 23 - vcipher 20, 20, 23 - vcipher 21, 21, 23 - vcipher 22, 22, 23 +.macro SAVE_VRS VRS OFFSET FRAME + stxv \VRS+32, \OFFSET(\FRAME) .endm -.macro Loop_aes_middle_1x - xxlor 19+32, 1, 1 - xxlor 20+32, 2, 2 - xxlor 21+32, 3, 3 - xxlor 22+32, 4, 4 - - vcipher 15, 15, 19 - vcipher 15, 15, 20 - vcipher 15, 15, 21 - vcipher 15, 15, 22 - - xxlor 19+32, 5, 5 - xxlor 20+32, 6, 6 - xxlor 21+32, 7, 7 - xxlor 22+32, 8, 8 - - vcipher 15, 15, 19 - vcipher 15, 15, 20 - vcipher 15, 15, 21 - vcipher 15, 15, 22 - - xxlor 19+32, 9, 9 - vcipher 15, 15, 19 +.macro RESTORE_GPR GPR OFFSET FRAME + ld \GPR,\OFFSET(\FRAME) .endm - # - # Compute 4x hash values based on Karatsuba method. - # -.macro ppc_aes_gcm_ghash - vxor 15, 15, 0 - - vpmsumd 23, 12, 15 # H4.L * X.L - vpmsumd 24, 9, 16 - vpmsumd 25, 6, 17 - vpmsumd 26, 3, 18 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 # L - - vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L - vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L - vpmsumd 26, 7, 17 - vpmsumd 27, 4, 18 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 # M - - # sum hash and reduction with H Poly - vpmsumd 28, 23, 2 # reduction - - vxor 29, 29, 29 - vsldoi 26, 24, 29, 8 # mL - vsldoi 29, 29, 24, 8 # mH - vxor 23, 23, 26 # mL + L - - vsldoi 23, 23, 23, 8 # swap - vxor 23, 23, 28 - - vpmsumd 24, 14, 15 # H4.H * X.H - vpmsumd 25, 11, 16 - vpmsumd 26, 8, 17 - vpmsumd 27, 5, 18 - - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 - - vxor 24, 24, 29 - - # sum hash and reduction with H Poly - vsldoi 27, 23, 23, 8 # swap - vpmsumd 23, 23, 2 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 # update hash - +.macro RESTORE_VRS VRS OFFSET FRAME + lxv \VRS+32, \OFFSET(\FRAME) .endm - # - # Combine two 4x ghash - # v15 - v22 - input blocks - # -.macro ppc_aes_gcm_ghash2_4x - # first 4x hash - vxor 15, 15, 0 # Xi + X - - vpmsumd 23, 12, 15 # H4.L * X.L - vpmsumd 24, 9, 16 - vpmsumd 25, 6, 17 - vpmsumd 26, 3, 18 - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 # L - - vpmsumd 24, 13, 15 # H4.L * X.H + H4.H * X.L - vpmsumd 25, 10, 16 # H3.L * X1.H + H3.H * X1.L - vpmsumd 26, 7, 17 - vpmsumd 27, 4, 18 - - vxor 24, 24, 25 - vxor 24, 24, 26 - - # sum hash and reduction with H Poly - vpmsumd 28, 23, 2 # reduction - - vxor 29, 29, 29 - - vxor 24, 24, 27 # M - vsldoi 26, 24, 29, 8 # mL - vsldoi 29, 29, 24, 8 # mH - vxor 23, 23, 26 # mL + L - - vsldoi 23, 23, 23, 8 # swap - vxor 23, 23, 28 +.macro SAVE_REGS + mflr 0 + std 0, 16(1) + stdu 1,-512(1) + + SAVE_GPR 14, 112, 1 + SAVE_GPR 15, 120, 1 + SAVE_GPR 16, 128, 1 + SAVE_GPR 17, 136, 1 + SAVE_GPR 18, 144, 1 + SAVE_GPR 19, 152, 1 + SAVE_GPR 20, 160, 1 + SAVE_GPR 21, 168, 1 + SAVE_GPR 22, 176, 1 + SAVE_GPR 23, 184, 1 + SAVE_GPR 24, 192, 1 + + addi 9, 1, 256 + SAVE_VRS 20, 0, 9 + SAVE_VRS 21, 16, 9 + SAVE_VRS 22, 32, 9 + SAVE_VRS 23, 48, 9 + SAVE_VRS 24, 64, 9 + SAVE_VRS 25, 80, 9 + SAVE_VRS 26, 96, 9 + SAVE_VRS 27, 112, 9 + SAVE_VRS 28, 128, 9 + SAVE_VRS 29, 144, 9 + SAVE_VRS 30, 160, 9 + SAVE_VRS 31, 176, 9 +.endm # SAVE_REGS - vpmsumd 24, 14, 15 # H4.H * X.H - vpmsumd 25, 11, 16 - vpmsumd 26, 8, 17 - vpmsumd 27, 5, 18 +.macro RESTORE_REGS + addi 9, 1, 256 + RESTORE_VRS 20, 0, 9 + RESTORE_VRS 21, 16, 9 + RESTORE_VRS 22, 32, 9 + RESTORE_VRS 23, 48, 9 + RESTORE_VRS 24, 64, 9 + RESTORE_VRS 25, 80, 9 + RESTORE_VRS 26, 96, 9 + RESTORE_VRS 27, 112, 9 + RESTORE_VRS 28, 128, 9 + RESTORE_VRS 29, 144, 9 + RESTORE_VRS 30, 160, 9 + RESTORE_VRS 31, 176, 9 + + RESTORE_GPR 14, 112, 1 + RESTORE_GPR 15, 120, 1 + RESTORE_GPR 16, 128, 1 + RESTORE_GPR 17, 136, 1 + RESTORE_GPR 18, 144, 1 + RESTORE_GPR 19, 152, 1 + RESTORE_GPR 20, 160, 1 + RESTORE_GPR 21, 168, 1 + RESTORE_GPR 22, 176, 1 + RESTORE_GPR 23, 184, 1 + RESTORE_GPR 24, 192, 1 + + addi 1, 1, 512 + ld 0, 16(1) + mtlr 0 +.endm # RESTORE_REGS + +# 4x loops +.macro AES_CIPHER_4x _VCIPHER ST r + \_VCIPHER \ST, \ST, \r + \_VCIPHER \ST+1, \ST+1, \r + \_VCIPHER \ST+2, \ST+2, \r + \_VCIPHER \ST+3, \ST+3, \r +.endm - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 # H +# 8x loops +.macro AES_CIPHER_8x _VCIPHER ST r + \_VCIPHER \ST, \ST, \r + \_VCIPHER \ST+1, \ST+1, \r + \_VCIPHER \ST+2, \ST+2, \r + \_VCIPHER \ST+3, \ST+3, \r + \_VCIPHER \ST+4, \ST+4, \r + \_VCIPHER \ST+5, \ST+5, \r + \_VCIPHER \ST+6, \ST+6, \r + \_VCIPHER \ST+7, \ST+7, \r +.endm - vxor 24, 24, 29 # H + mH +.macro LOOP_8AES_STATE + xxlor 32+23, 1, 1 + xxlor 32+24, 2, 2 + xxlor 32+25, 3, 3 + xxlor 32+26, 4, 4 + AES_CIPHER_8x vcipher, 15, 23 + AES_CIPHER_8x vcipher, 15, 24 + AES_CIPHER_8x vcipher, 15, 25 + AES_CIPHER_8x vcipher, 15, 26 + xxlor 32+23, 5, 5 + xxlor 32+24, 6, 6 + xxlor 32+25, 7, 7 + xxlor 32+26, 8, 8 + AES_CIPHER_8x vcipher, 15, 23 + AES_CIPHER_8x vcipher, 15, 24 + AES_CIPHER_8x vcipher, 15, 25 + AES_CIPHER_8x vcipher, 15, 26 +.endm - # sum hash and reduction with H Poly - vsldoi 27, 23, 23, 8 # swap - vpmsumd 23, 23, 2 - vxor 27, 27, 24 - vxor 27, 23, 27 # 1st Xi - - # 2nd 4x hash - vpmsumd 24, 9, 20 - vpmsumd 25, 6, 21 - vpmsumd 26, 3, 22 - vxor 19, 19, 27 # Xi + X - vpmsumd 23, 12, 19 # H4.L * X.L - - vxor 23, 23, 24 - vxor 23, 23, 25 - vxor 23, 23, 26 # L - - vpmsumd 24, 13, 19 # H4.L * X.H + H4.H * X.L - vpmsumd 25, 10, 20 # H3.L * X1.H + H3.H * X1.L - vpmsumd 26, 7, 21 - vpmsumd 27, 4, 22 - - vxor 24, 24, 25 - vxor 24, 24, 26 +# +# PPC_GHASH4x(H, S1, S2, S3, S4): Compute 4x hash values based on Karatsuba method. +# H: returning digest +# S#: states +# +# S1 should xor with the previous digest +# +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# Scratch: v23 - v29 +# +.macro PPC_GHASH4x H S1 S2 S3 S4 + + vpmsumd 23, 12, \S1 # H4.L * X.L + vpmsumd 24, 9, \S2 + vpmsumd 25, 6, \S3 + vpmsumd 26, 3, \S4 + + vpmsumd 27, 13, \S1 # H4.L * X.H + H4.H * X.L + vpmsumd 28, 10, \S2 # H3.L * X1.H + H3.H * X1.L + + vxor 23, 23, 24 + vxor 23, 23, 25 + vxor 23, 23, 26 # L + + vxor 24, 27, 28 + vpmsumd 25, 7, \S3 + vpmsumd 26, 4, \S4 + + vxor 24, 24, 25 + vxor 24, 24, 26 # M # sum hash and reduction with H Poly - vpmsumd 28, 23, 2 # reduction - - vxor 29, 29, 29 + vpmsumd 28, 23, 2 # reduction - vxor 24, 24, 27 # M - vsldoi 26, 24, 29, 8 # mL - vsldoi 29, 29, 24, 8 # mH - vxor 23, 23, 26 # mL + L + vxor 1, 1, 1 + vsldoi 25, 24, 1, 8 # mL + vsldoi 1, 1, 24, 8 # mH + vxor 23, 23, 25 # mL + L - vsldoi 23, 23, 23, 8 # swap - vxor 23, 23, 28 + # This performs swap and xor like, + # vsldoi 23, 23, 23, 8 # swap + # vxor 23, 23, 28 + xxlor 32+25, 10, 10 + vpermxor 23, 23, 28, 25 - vpmsumd 24, 14, 19 # H4.H * X.H - vpmsumd 25, 11, 20 - vpmsumd 26, 8, 21 - vpmsumd 27, 5, 22 + vpmsumd 26, 14, \S1 # H4.H * X.H + vpmsumd 27, 11, \S2 + vpmsumd 28, 8, \S3 + vpmsumd 29, 5, \S4 - vxor 24, 24, 25 - vxor 24, 24, 26 - vxor 24, 24, 27 # H + vxor 24, 26, 27 + vxor 24, 24, 28 + vxor 24, 24, 29 - vxor 24, 24, 29 # H + mH + vxor 24, 24, 1 # sum hash and reduction with H Poly - vsldoi 27, 23, 23, 8 # swap - vpmsumd 23, 23, 2 - vxor 27, 27, 24 - vxor 23, 23, 27 - - xxlor 32, 23+32, 23+32 # update hash - + vsldoi 25, 23, 23, 8 # swap + vpmsumd 23, 23, 2 + vxor 27, 25, 24 + vxor \H, 23, 27 .endm - # - # Compute update single hash - # -.macro ppc_update_hash_1x - vxor 28, 28, 0 - - vxor 19, 19, 19 +# +# Compute update single ghash +# scratch: v1, v22..v27 +# +.macro PPC_GHASH1x H S1 - vpmsumd 22, 3, 28 # L - vpmsumd 23, 4, 28 # M - vpmsumd 24, 5, 28 # H + vxor 1, 1, 1 - vpmsumd 27, 22, 2 # reduction + vpmsumd 22, 3, \S1 # L + vpmsumd 23, 4, \S1 # M + vpmsumd 24, 5, \S1 # H - vsldoi 25, 23, 19, 8 # mL - vsldoi 26, 19, 23, 8 # mH - vxor 22, 22, 25 # LL + LL - vxor 24, 24, 26 # HH + HH + vpmsumd 27, 22, 2 # reduction - vsldoi 22, 22, 22, 8 # swap - vxor 22, 22, 27 + vsldoi 25, 23, 1, 8 # mL + vsldoi 26, 1, 23, 8 # mH + vxor 22, 22, 25 # LL + LL + vxor 24, 24, 26 # HH + HH - vsldoi 20, 22, 22, 8 # swap - vpmsumd 22, 22, 2 # reduction - vxor 20, 20, 24 - vxor 22, 22, 20 + xxlor 32+25, 10, 10 + vpermxor 22, 22, 27, 25 - vmr 0, 22 # update hash - -.endm - -.macro SAVE_REGS - stdu 1,-640(1) - mflr 0 - - std 14,112(1) - std 15,120(1) - std 16,128(1) - std 17,136(1) - std 18,144(1) - std 19,152(1) - std 20,160(1) - std 21,168(1) - li 9, 256 - stvx 20, 9, 1 - addi 9, 9, 16 - stvx 21, 9, 1 - addi 9, 9, 16 - stvx 22, 9, 1 - addi 9, 9, 16 - stvx 23, 9, 1 - addi 9, 9, 16 - stvx 24, 9, 1 - addi 9, 9, 16 - stvx 25, 9, 1 - addi 9, 9, 16 - stvx 26, 9, 1 - addi 9, 9, 16 - stvx 27, 9, 1 - addi 9, 9, 16 - stvx 28, 9, 1 - addi 9, 9, 16 - stvx 29, 9, 1 - addi 9, 9, 16 - stvx 30, 9, 1 - addi 9, 9, 16 - stvx 31, 9, 1 - stxv 14, 464(1) - stxv 15, 480(1) - stxv 16, 496(1) - stxv 17, 512(1) - stxv 18, 528(1) - stxv 19, 544(1) - stxv 20, 560(1) - stxv 21, 576(1) - stxv 22, 592(1) - std 0, 656(1) -.endm - -.macro RESTORE_REGS - lxv 14, 464(1) - lxv 15, 480(1) - lxv 16, 496(1) - lxv 17, 512(1) - lxv 18, 528(1) - lxv 19, 544(1) - lxv 20, 560(1) - lxv 21, 576(1) - lxv 22, 592(1) - li 9, 256 - lvx 20, 9, 1 - addi 9, 9, 16 - lvx 21, 9, 1 - addi 9, 9, 16 - lvx 22, 9, 1 - addi 9, 9, 16 - lvx 23, 9, 1 - addi 9, 9, 16 - lvx 24, 9, 1 - addi 9, 9, 16 - lvx 25, 9, 1 - addi 9, 9, 16 - lvx 26, 9, 1 - addi 9, 9, 16 - lvx 27, 9, 1 - addi 9, 9, 16 - lvx 28, 9, 1 - addi 9, 9, 16 - lvx 29, 9, 1 - addi 9, 9, 16 - lvx 30, 9, 1 - addi 9, 9, 16 - lvx 31, 9, 1 - - ld 0, 656(1) - ld 14,112(1) - ld 15,120(1) - ld 16,128(1) - ld 17,136(1) - ld 18,144(1) - ld 19,152(1) - ld 20,160(1) - ld 21,168(1) - - mtlr 0 - addi 1, 1, 640 + vsldoi 23, 22, 22, 8 # swap + vpmsumd 22, 22, 2 # reduction + vxor 23, 23, 24 + vxor \H, 22, 23 .endm +# +# LOAD_HASH_TABLE +# Xi = v0 +# H Poly = v2 +# Hash keys = v3 - v14 +# .macro LOAD_HASH_TABLE # Load Xi lxvb16x 32, 0, 8 # load Xi @@ -557,657 +298,434 @@ lxvd2x 14+32, 10, 8 # H^4h .endm - # - # aes_p10_gcm_encrypt (const void *inp, void *out, size_t len, - # const char *rk, unsigned char iv[16], void *Xip); - # - # r3 - inp - # r4 - out - # r5 - len - # r6 - AES round keys - # r7 - iv and other data - # r8 - Xi, HPoli, hash keys - # - # rounds is at offset 240 in rk - # Xi is at 0 in gcm_table (Xip). - # -_GLOBAL(aes_p10_gcm_encrypt) -.align 5 - - SAVE_REGS - - LOAD_HASH_TABLE - - # initialize ICB: GHASH( IV ), IV - r7 - lxvb16x 30+32, 0, 7 # load IV - v30 - - mr 12, 5 # length - li 11, 0 # block index - - # counter 1 - vxor 31, 31, 31 - vspltisb 22, 1 - vsldoi 31, 31, 22,1 # counter 1 - - # load round key to VSR - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) - - # load rounds - 10 (128), 12 (192), 14 (256) - lwz 9,240(6) - - # - # vxor state, state, w # addroundkey - xxlor 32+29, 0, 0 - vxor 15, 30, 29 # IV + round key - add round key 0 - - cmpdi 9, 10 - beq Loop_aes_gcm_8x - - # load 2 more round keys (v11, v12) - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) - - cmpdi 9, 12 - beq Loop_aes_gcm_8x - - # load 2 more round keys (v11, v12, v13, v14) - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq Loop_aes_gcm_8x - - b aes_gcm_out - -.align 5 -Loop_aes_gcm_8x: - mr 14, 3 - mr 9, 4 - - # - # check partial block - # -Continue_partial_check: - ld 15, 56(7) - cmpdi 15, 0 - beq Continue - bgt Final_block - cmpdi 15, 16 - blt Final_block - -Continue: - # n blcoks - li 10, 128 - divdu 10, 12, 10 # n 128 bytes-blocks - cmpdi 10, 0 - beq Loop_last_block - - vaddudm 30, 30, 31 # IV + counter - vxor 16, 30, 29 - vaddudm 30, 30, 31 - vxor 17, 30, 29 - vaddudm 30, 30, 31 - vxor 18, 30, 29 - vaddudm 30, 30, 31 - vxor 19, 30, 29 - vaddudm 30, 30, 31 - vxor 20, 30, 29 - vaddudm 30, 30, 31 - vxor 21, 30, 29 - vaddudm 30, 30, 31 - vxor 22, 30, 29 - - mtctr 10 - - li 15, 16 - li 16, 32 - li 17, 48 - li 18, 64 - li 19, 80 - li 20, 96 - li 21, 112 - - lwz 10, 240(6) - -Loop_8x_block: - - lxvb16x 15, 0, 14 # load block - lxvb16x 16, 15, 14 # load block - lxvb16x 17, 16, 14 # load block - lxvb16x 18, 17, 14 # load block - lxvb16x 19, 18, 14 # load block - lxvb16x 20, 19, 14 # load block - lxvb16x 21, 20, 14 # load block - lxvb16x 22, 21, 14 # load block - addi 14, 14, 128 - - Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_ghash - - # 192 bits - xxlor 24+32, 11, 11 - - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 - vcipher 19, 19, 23 - vcipher 20, 20, 23 - vcipher 21, 21, 23 - vcipher 22, 22, 23 - - vcipher 15, 15, 24 - vcipher 16, 16, 24 - vcipher 17, 17, 24 - vcipher 18, 18, 24 - vcipher 19, 19, 24 - vcipher 20, 20, 24 - vcipher 21, 21, 24 - vcipher 22, 22, 24 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_ghash - - # 256 bits - xxlor 24+32, 13, 13 - - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 - vcipher 19, 19, 23 - vcipher 20, 20, 23 - vcipher 21, 21, 23 - vcipher 22, 22, 23 - - vcipher 15, 15, 24 - vcipher 16, 16, 24 - vcipher 17, 17, 24 - vcipher 18, 18, 24 - vcipher 19, 19, 24 - vcipher 20, 20, 24 - vcipher 21, 21, 24 - vcipher 22, 22, 24 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_ghash - b aes_gcm_out - -Do_next_ghash: - - # - # last round - vcipherlast 15, 15, 23 - vcipherlast 16, 16, 23 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 # store output - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 # store output - - vcipherlast 17, 17, 23 - vcipherlast 18, 18, 23 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 # store output - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 # store output - - vcipherlast 19, 19, 23 - vcipherlast 20, 20, 23 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 # store output - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 # store output - - vcipherlast 21, 21, 23 - vcipherlast 22, 22, 23 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 # store output - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 # store output - - addi 9, 9, 128 - - # ghash here - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - vaddudm 30, 30, 31 # IV + counter - vmr 29, 30 - vxor 15, 30, 27 # add round key - vaddudm 30, 30, 31 - vxor 16, 30, 27 - vaddudm 30, 30, 31 - vxor 17, 30, 27 - vaddudm 30, 30, 31 - vxor 18, 30, 27 - vaddudm 30, 30, 31 - vxor 19, 30, 27 - vaddudm 30, 30, 31 - vxor 20, 30, 27 - vaddudm 30, 30, 31 - vxor 21, 30, 27 - vaddudm 30, 30, 31 - vxor 22, 30, 27 - - addi 12, 12, -128 - addi 11, 11, 128 - - bdnz Loop_8x_block - - vmr 30, 29 - stxvb16x 30+32, 0, 7 # update IV - -Loop_last_block: - cmpdi 12, 0 - beq aes_gcm_out - - # loop last few blocks +################################################################################ +# Compute AES and ghash one block at a time. +# r23: AES rounds +# v30: current IV +# vs0: roundkey 0 +# +################################################################################ +SYM_FUNC_START_LOCAL(aes_gcm_crypt_1x) + + cmpdi 5, 16 + bge __More_1x + blr +__More_1x: li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10, 240(6) - - cmpdi 12, 16 - blt Final_block - -Next_rem_block: - lxvb16x 15, 0, 14 # load block - - Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_1x - - # 192 bits - xxlor 24+32, 11, 11 - - vcipher 15, 15, 23 - vcipher 15, 15, 24 - - xxlor 23+32, 12, 12 + divdu 12, 5, 10 + + xxlxor 32+15, 32+30, 0 + + # Pre-load 8 AES rounds to scratch vectors. + xxlor 32+16, 1, 1 + xxlor 32+17, 2, 2 + xxlor 32+18, 3, 3 + xxlor 32+19, 4, 4 + xxlor 32+20, 5, 5 + xxlor 32+21, 6, 6 + xxlor 32+28, 7, 7 + xxlor 32+29, 8, 8 + lwz 23, 240(6) # n rounds + addi 22, 23, -9 # remaing AES rounds - cmpdi 10, 12 - beq Do_next_1x - - # 256 bits - xxlor 24+32, 13, 13 - - vcipher 15, 15, 23 - vcipher 15, 15, 24 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_1x - -Do_next_1x: - vcipherlast 15, 15, 23 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 # store output - addi 14, 14, 16 - addi 9, 9, 16 - - vmr 28, 15 - ppc_update_hash_1x - - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - vaddudm 30, 30, 31 # IV + counter - vxor 15, 30, 19 # add round key - - bdnz Next_rem_block - - li 15, 0 - std 15, 56(7) # clear partial? - stxvb16x 30+32, 0, 7 # update IV cmpdi 12, 0 - beq aes_gcm_out - -Final_block: - lwz 10, 240(6) - Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x - - # 192 bits - xxlor 24+32, 11, 11 - - vcipher 15, 15, 23 - vcipher 15, 15, 24 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_final_1x - - # 256 bits - xxlor 24+32, 13, 13 - - vcipher 15, 15, 23 - vcipher 15, 15, 24 + bgt __Loop_1x + blr - xxlor 23+32, 14, 14 +__Loop_1x: + mtctr 22 + addi 10, 6, 144 + vcipher 15, 15, 16 + vcipher 15, 15, 17 + vcipher 15, 15, 18 + vcipher 15, 15, 19 + vcipher 15, 15, 20 + vcipher 15, 15, 21 + vcipher 15, 15, 28 + vcipher 15, 15, 29 - cmpdi 10, 14 - beq Do_final_1x +__Loop_aes_1state: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_1state + lxv 32+1, 0(10) # last round key + lxvb16x 11, 0, 14 # load input block + vcipherlast 15, 15, 1 + + xxlxor 32+15, 32+15, 11 + stxvb16x 32+15, 0, 9 # store output + addi 14, 14, 16 + addi 9, 9, 16 -Do_final_1x: - vcipherlast 15, 15, 23 + cmpdi 24, 0 # decrypt? + bne __Encrypt_1x + xxlor 15+32, 11, 11 +__Encrypt_1x: + vxor 15, 15, 0 + PPC_GHASH1x 0, 15 - # check partial block - li 21, 0 # encrypt - ld 15, 56(7) # partial? - cmpdi 15, 0 - beq Normal_block - bl Do_partial_block + addi 5, 5, -16 + addi 11, 11, 16 + vadduwm 30, 30, 31 # IV + counter + xxlxor 32+15, 32+30, 0 + addi 12, 12, -1 cmpdi 12, 0 - ble aes_gcm_out + bgt __Loop_1x - b Continue_partial_check - -Normal_block: - lxvb16x 15, 0, 14 # load last block - xxlxor 47, 47, 15 - - # create partial block mask - li 15, 16 - sub 15, 15, 12 # index to the mask - - vspltisb 16, -1 # first 16 bytes - 0xffff...ff - vspltisb 17, 0 # second 16 bytes - 0x0000...00 - li 10, 192 - stvx 16, 10, 1 + stxvb16x 32+30, 0, 7 # update IV + stxvb16x 32+0, 0, 8 # update Xi + blr +SYM_FUNC_END(aes_gcm_crypt_1x) + +################################################################################ +# Process a normal partial block when we come here. +# Compute partial mask, Load and store partial block to stack. +# Update partial_len and pblock. +# pblock is (encrypted ^ AES state) for encrypt +# and (input ^ AES state) for decrypt. +# +################################################################################ +SYM_FUNC_START_LOCAL(__Process_partial) + + # create partial mask + vspltisb 16, -1 + li 12, 16 + sub 12, 12, 5 + sldi 12, 12, 3 + mtvsrdd 32+17, 0, 12 + vslo 16, 16, 17 # partial block mask + + lxvb16x 11, 0, 14 # load partial block + xxland 11, 11, 32+16 + + # AES crypt partial + xxlxor 32+15, 32+30, 0 + lwz 23, 240(6) # n rounds + addi 22, 23, -1 # loop - 1 + mtctr 22 + addi 10, 6, 16 + +__Loop_aes_pstate: + lxv 32+1, 0(10) + vcipher 15, 15, 1 addi 10, 10, 16 - stvx 17, 10, 1 - - addi 10, 1, 192 - lxvb16x 16, 15, 10 # load partial block mask - xxland 47, 47, 16 - - vmr 28, 15 - ppc_update_hash_1x + bdnz __Loop_aes_pstate + lxv 32+1, 0(10) # last round key + vcipherlast 15, 15, 1 - # * should store only the remaining bytes. - bl Write_partial_block - - stxvb16x 30+32, 0, 7 # update IV - std 12, 56(7) # update partial? - li 16, 16 + xxlxor 32+15, 32+15, 11 + vand 15, 15, 16 - stxvb16x 32, 0, 8 # write out Xi - stxvb16x 32, 16, 8 # write out Xi - b aes_gcm_out - - # - # Compute data mask - # -.macro GEN_MASK _mask _start _end - vspltisb 16, -1 # first 16 bytes - 0xffff...ff - vspltisb 17, 0 # second 16 bytes - 0x0000...00 - li 10, 192 - stxvb16x 17+32, 10, 1 - add 10, 10, \_start - stxvb16x 16+32, 10, 1 - add 10, 10, \_end - stxvb16x 17+32, 10, 1 - - addi 10, 1, 192 - lxvb16x \_mask, 0, 10 # load partial block mask -.endm + # AES crypt output v15 + # Write partial + li 10, 224 + stxvb16x 15+32, 10, 1 # write v15 to stack + addi 10, 1, 223 + addi 12, 9, -1 + mtctr 5 # partial block len +__Write_partial: + lbzu 22, 1(10) + stbu 22, 1(12) + bdnz __Write_partial + + cmpdi 24, 0 # decrypt? + bne __Encrypt_partial + xxlor 32+15, 11, 11 # decrypt using the input block +__Encrypt_partial: + #vxor 15, 15, 0 # ^ previous hash + #PPC_GHASH1x 0, 15 + + add 14, 14, 5 + add 9, 9, 5 + std 5, 56(7) # update partial + sub 11, 11, 5 + li 5, 0 # done last byte - # - # Handle multiple partial blocks for encrypt and decrypt - # operations. - # -SYM_FUNC_START_LOCAL(Do_partial_block) - add 17, 15, 5 - cmpdi 17, 16 - bgt Big_block - GEN_MASK 18, 15, 5 - b _Partial -SYM_FUNC_END(Do_partial_block) -Big_block: + # + # Don't increase IV since this is the last partial. + # It should get updated in gcm_update if no more data blocks. + #vadduwm 30, 30, 31 # increase IV + stxvb16x 32+30, 0, 7 # update IV + li 10, 64 + stxvb16x 32+0, 0, 8 # Update X1 + stxvb16x 32+15, 10, 7 # Update pblock + blr +SYM_FUNC_END(__Process_partial) + +################################################################################ +# Combine partial blocks and ghash when we come here. +# +# The partial block has to be shifted to the right location to encrypt/decrypt +# and compute ghash if combing the previous partial block is needed. +# - Compute ghash for a full block. Clear Partial_len and pblock. Update IV. +# Write Xi. +# - Don't compute ghash if not full block. gcm_update will take care of it +# is the last block. Update Partial_len and pblock. +# +################################################################################ +SYM_FUNC_START_LOCAL(__Combine_partial) + + ld 12, 56(7) + mr 21, 5 # these bytes to be processed + + li 17, 0 li 16, 16 - GEN_MASK 18, 15, 16 - -_Partial: - lxvb16x 17+32, 0, 14 # load last block - sldi 16, 15, 3 - mtvsrdd 32+16, 0, 16 - vsro 17, 17, 16 - xxlxor 47, 47, 17+32 - xxland 47, 47, 18 - - vxor 0, 0, 0 # clear Xi - vmr 28, 15 - - cmpdi 21, 0 # encrypt/decrypt ops? - beq Skip_decrypt - xxland 32+28, 32+17, 18 - -Skip_decrypt: - - ppc_update_hash_1x + sub 22, 16, 12 # bytes to complete a block + sub 17, 22, 5 # remaining bytes in a block + cmpdi 5, 16 + ble __Inp_msg_less16 + li 17, 0 + mr 21, 22 + b __Combine_continue +__Inp_msg_less16: + cmpd 22, 5 + bgt __Combine_continue + li 17, 0 + mr 21, 22 # these bytes to be processed + +__Combine_continue: + # load msg and shift to the proper location and mask + vspltisb 16, -1 + sldi 15, 12, 3 + mtvsrdd 32+17, 0, 15 + vslo 16, 16, 17 + vsro 16, 16, 17 + sldi 15, 17, 3 + mtvsrdd 32+17, 0, 15 + vsro 16, 16, 17 + vslo 16, 16, 17 # mask + + lxvb16x 32+19, 0, 14 # load partial block + sldi 15, 12, 3 + mtvsrdd 32+17, 0, 15 + vsro 19, 19, 17 # 0x00..xxxx??..?? + sldi 15, 17, 3 + mtvsrdd 32+17, 0, 15 + vsro 19, 19, 17 # 0x00..xxxx + vslo 19, 19, 17 # shift back to form 0x00..xxxx00..00 + + # AES crypt partial + xxlxor 32+15, 32+30, 0 + lwz 23, 240(6) # n rounds + addi 22, 23, -1 # loop - 1 + mtctr 22 + addi 10, 6, 16 + +__Loop_aes_cpstate: + lxv 32+1, 0(10) + vcipher 15, 15, 1 + addi 10, 10, 16 + bdnz __Loop_aes_cpstate + lxv 32+1, 0(10) # last round key + vcipherlast 15, 15, 1 - li 16, 16 - lxvb16x 32+29, 16, 8 - vxor 0, 0, 29 - stxvb16x 32, 0, 8 # save Xi - stxvb16x 32, 16, 8 # save Xi - - # store partial block - # loop the rest of the stream if any - sldi 16, 15, 3 - mtvsrdd 32+16, 0, 16 - vslo 15, 15, 16 - #stxvb16x 15+32, 0, 9 # last block + vxor 15, 15, 19 + vand 15, 15, 16 - li 16, 16 - sub 17, 16, 15 # 16 - partial - - add 16, 15, 5 - cmpdi 16, 16 - bgt Larger_16 - mr 17, 5 -Larger_16: - - # write partial - li 10, 192 - stxvb16x 15+32, 10, 1 # save current block - - addi 10, 9, -1 - addi 16, 1, 191 - mtctr 17 # move partial byte count - -Write_last_partial: - lbzu 18, 1(16) - stbu 18, 1(10) - bdnz Write_last_partial - # Complete loop partial - - add 14, 14, 17 - add 9, 9, 17 - sub 12, 12, 17 - add 11, 11, 17 - - add 15, 15, 5 - cmpdi 15, 16 - blt Save_partial - - vaddudm 30, 30, 31 - stxvb16x 30+32, 0, 7 # update IV - xxlor 32+29, 0, 0 - vxor 15, 30, 29 # IV + round key - add round key 0 - li 15, 0 - std 15, 56(7) # partial done - clear - b Partial_done -Save_partial: - std 15, 56(7) # partial - -Partial_done: + # AES crypt output v15 + # Write partial + li 10, 224 + stxvb16x 15+32, 10, 1 # write v15 to stack + addi 10, 1, 223 + add 10, 10, 12 # add offset + addi 15, 9, -1 + mtctr 21 # partial block len +__Write_combine_partial: + lbzu 22, 1(10) + stbu 22, 1(15) + bdnz __Write_combine_partial + + add 14, 14, 21 + add 11, 11, 21 + add 9, 9, 21 + sub 5, 5, 21 + + # Encrypt/Decrypt? + cmpdi 24, 0 # decrypt? + bne __Encrypt_combine_partial + vmr 15, 19 # decrypt using the input block + +__Encrypt_combine_partial: + # + # Update partial flag and combine ghash. +__Update_partial_ghash: + li 10, 64 + lxvb16x 32+17, 10, 7 # load previous pblock + add 12, 12, 21 # combined pprocessed + vxor 15, 15, 17 # combined pblock + + cmpdi 12, 16 + beq __Clear_partial_flag + std 12, 56(7) # update partial len + stxvb16x 32+15, 10, 7 # Update current pblock blr - # - # Write partial block - # r9 - output - # r12 - remaining bytes - # v15 - partial input data - # -SYM_FUNC_START_LOCAL(Write_partial_block) - li 10, 192 - stxvb16x 15+32, 10, 1 # last block - - addi 10, 9, -1 - addi 16, 1, 191 - - mtctr 12 # remaining bytes - li 15, 0 - -Write_last_byte: - lbzu 14, 1(16) - stbu 14, 1(10) - bdnz Write_last_byte +__Clear_partial_flag: + li 12, 0 + std 12, 56(7) + # Update IV and ghash here + vadduwm 30, 30, 31 # increase IV + stxvb16x 32+30, 0, 7 # update IV + + # v15 either is either (input blockor encrypted)^(AES state) + vxor 15, 15, 0 + PPC_GHASH1x 0, 15 + stxvb16x 32+0, 10, 7 # update pblock for debug? + stxvb16x 32+0, 0, 8 # update Xi blr -SYM_FUNC_END(Write_partial_block) +SYM_FUNC_END(__Combine_partial) -aes_gcm_out: - # out = state - stxvb16x 32, 0, 8 # write out Xi - add 3, 11, 12 # return count +################################################################################ +# gcm_update(iv, Xi) - compute last hash +# +################################################################################ +SYM_FUNC_START(gcm_update) - RESTORE_REGS - blr + ld 10, 56(3) + cmpdi 10, 0 + beq __no_update - # - # 8x Decrypt - # -_GLOBAL(aes_p10_gcm_decrypt) -.align 5 + lxvb16x 32, 0, 4 # load Xi + # load Hash - h^4, h^3, h^2, h + li 10, 32 + lxvd2x 2+32, 10, 4 # H Poli + li 10, 48 + lxvd2x 3+32, 10, 4 # Hl + li 10, 64 + lxvd2x 4+32, 10, 4 # H + li 10, 80 + lxvd2x 5+32, 10, 4 # Hh + + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + + li 9, 64 + lxvb16x 32+6, 9, 3 # load pblock + vxor 6, 6, 0 + + vxor 1, 1, 1 + vpmsumd 12, 3, 6 # L + vpmsumd 13, 4, 6 # M + vpmsumd 14, 5, 6 # H + vpmsumd 17, 12, 2 # reduction + vsldoi 15, 13, 1, 8 # mL + vsldoi 16, 1, 13, 8 # mH + vxor 12, 12, 15 # LL + LL + vxor 14, 14, 16 # HH + HH + xxlor 32+15, 10, 10 + vpermxor 12, 12, 17, 15 + vsldoi 13, 12, 12, 8 # swap + vpmsumd 12, 12, 2 # reduction + vxor 13, 13, 14 + vxor 7, 12, 13 + + #vxor 0, 0, 0 + #stxvb16x 32+0, 9, 3 + li 10, 0 + std 10, 56(3) + stxvb16x 32+7, 0, 4 + +__no_update: + blr +SYM_FUNC_END(gcm_update) + +################################################################################ +# aes_p10_gcm_encrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# +# r3 - inp +# r4 - out +# r5 - len +# r6 - AES round keys +# r7 - iv and other data +# r8 - Xi, HPoli, hash keys +# +# rounds is at offset 240 in rk +# Xi is at 0 in gcm_table (Xip). +# +################################################################################ +SYM_FUNC_START(aes_p10_gcm_encrypt) + + cmpdi 5, 0 + ble __Invalid_msg_len SAVE_REGS - LOAD_HASH_TABLE # initialize ICB: GHASH( IV ), IV - r7 lxvb16x 30+32, 0, 7 # load IV - v30 - mr 12, 5 # length - li 11, 0 # block index + mr 14, 3 + mr 9, 4 # counter 1 vxor 31, 31, 31 vspltisb 22, 1 vsldoi 31, 31, 22,1 # counter 1 - # load round key to VSR - lxv 0, 0(6) - lxv 1, 0x10(6) - lxv 2, 0x20(6) - lxv 3, 0x30(6) - lxv 4, 0x40(6) - lxv 5, 0x50(6) - lxv 6, 0x60(6) - lxv 7, 0x70(6) - lxv 8, 0x80(6) - lxv 9, 0x90(6) - lxv 10, 0xa0(6) + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + + # load 9 round keys to VSR + lxv 0, 0(6) # round key 0 + lxv 1, 16(6) # round key 1 + lxv 2, 32(6) # round key 2 + lxv 3, 48(6) # round key 3 + lxv 4, 64(6) # round key 4 + lxv 5, 80(6) # round key 5 + lxv 6, 96(6) # round key 6 + lxv 7, 112(6) # round key 7 + lxv 8, 128(6) # round key 8 # load rounds - 10 (128), 12 (192), 14 (256) - lwz 9,240(6) + lwz 23, 240(6) # n rounds + li 24, 1 # encrypt +__Process_encrypt: # - # vxor state, state, w # addroundkey - xxlor 32+29, 0, 0 - vxor 15, 30, 29 # IV + round key - add round key 0 - - cmpdi 9, 10 - beq Loop_aes_gcm_8x_dec - - # load 2 more round keys (v11, v12) - lxv 11, 0xb0(6) - lxv 12, 0xc0(6) - - cmpdi 9, 12 - beq Loop_aes_gcm_8x_dec - - # load 2 more round keys (v11, v12, v13, v14) - lxv 13, 0xd0(6) - lxv 14, 0xe0(6) - cmpdi 9, 14 - beq Loop_aes_gcm_8x_dec + # Process different blocks + # + ld 12, 56(7) + cmpdi 12, 0 + bgt __Do_combine_enc + cmpdi 5, 128 + blt __Process_more_enc + +# +# Process 8x AES/GCM blocks +# +__Process_8x_enc: + # 8x blcoks + li 10, 128 + divdu 12, 5, 10 # n 128 bytes-blocks - b aes_gcm_out + addi 12, 12, -1 # loop - 1 -.align 5 -Loop_aes_gcm_8x_dec: - mr 14, 3 - mr 9, 4 + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state - # - # check partial block - # -Continue_partial_check_dec: - ld 15, 56(7) - cmpdi 15, 0 - beq Continue_dec - bgt Final_block_dec - cmpdi 15, 16 - blt Final_block_dec - -Continue_dec: - # n blcoks - li 10, 128 - divdu 10, 12, 10 # n 128 bytes-blocks - cmpdi 10, 0 - beq Loop_last_block_dec - - vaddudm 30, 30, 31 # IV + counter - vxor 16, 30, 29 - vaddudm 30, 30, 31 - vxor 17, 30, 29 - vaddudm 30, 30, 31 - vxor 18, 30, 29 - vaddudm 30, 30, 31 - vxor 19, 30, 29 - vaddudm 30, 30, 31 - vxor 20, 30, 29 - vaddudm 30, 30, 31 - vxor 21, 30, 29 - vaddudm 30, 30, 31 - vxor 22, 30, 29 - - mtctr 10 + # vxor state, state, w # addroundkey + xxlor 32+29, 0, 0 + vxor 15, 15, 29 # IV + round key - add round key 0 + vxor 16, 16, 29 + vxor 17, 17, 29 + vxor 18, 18, 29 + vxor 19, 19, 29 + vxor 20, 20, 29 + vxor 21, 21, 29 + vxor 22, 22, 29 li 15, 16 li 16, 32 @@ -1217,305 +735,502 @@ Continue_dec: li 20, 96 li 21, 112 - lwz 10, 240(6) - -Loop_8x_block_dec: - - lxvb16x 15, 0, 14 # load block - lxvb16x 16, 15, 14 # load block - lxvb16x 17, 16, 14 # load block - lxvb16x 18, 17, 14 # load block - lxvb16x 19, 18, 14 # load block - lxvb16x 20, 19, 14 # load block - lxvb16x 21, 20, 14 # load block - lxvb16x 22, 21, 14 # load block - addi 14, 14, 128 - - Loop_aes_middle8x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_next_ghash_dec - - # 192 bits - xxlor 24+32, 11, 11 - - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 - vcipher 19, 19, 23 - vcipher 20, 20, 23 - vcipher 21, 21, 23 - vcipher 22, 22, 23 - - vcipher 15, 15, 24 - vcipher 16, 16, 24 - vcipher 17, 17, 24 - vcipher 18, 18, 24 - vcipher 19, 19, 24 - vcipher 20, 20, 24 - vcipher 21, 21, 24 - vcipher 22, 22, 24 - - xxlor 23+32, 12, 12 - - cmpdi 10, 12 - beq Do_next_ghash_dec - - # 256 bits - xxlor 24+32, 13, 13 - - vcipher 15, 15, 23 - vcipher 16, 16, 23 - vcipher 17, 17, 23 - vcipher 18, 18, 23 - vcipher 19, 19, 23 - vcipher 20, 20, 23 - vcipher 21, 21, 23 - vcipher 22, 22, 23 - - vcipher 15, 15, 24 - vcipher 16, 16, 24 - vcipher 17, 17, 24 - vcipher 18, 18, 24 - vcipher 19, 19, 24 - vcipher 20, 20, 24 - vcipher 21, 21, 24 - vcipher 22, 22, 24 - - xxlor 23+32, 14, 14 - - cmpdi 10, 14 - beq Do_next_ghash_dec - b aes_gcm_out + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 -Do_next_ghash_dec: + LOOP_8AES_STATE # process 8 AES keys - # - # last round - vcipherlast 15, 15, 23 - vcipherlast 16, 16, 23 - - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 # store output - xxlxor 48, 48, 16 - stxvb16x 48, 15, 9 # store output - - vcipherlast 17, 17, 23 - vcipherlast 18, 18, 23 - - xxlxor 49, 49, 17 - stxvb16x 49, 16, 9 # store output - xxlxor 50, 50, 18 - stxvb16x 50, 17, 9 # store output - - vcipherlast 19, 19, 23 - vcipherlast 20, 20, 23 - - xxlxor 51, 51, 19 - stxvb16x 51, 18, 9 # store output - xxlxor 52, 52, 20 - stxvb16x 52, 19, 9 # store output - - vcipherlast 21, 21, 23 - vcipherlast 22, 22, 23 - - xxlxor 53, 53, 21 - stxvb16x 53, 20, 9 # store output - xxlxor 54, 54, 22 - stxvb16x 54, 21, 9 # store output - - addi 9, 9, 128 - - xxlor 15+32, 15, 15 - xxlor 16+32, 16, 16 - xxlor 17+32, 17, 17 - xxlor 18+32, 18, 18 - xxlor 19+32, 19, 19 - xxlor 20+32, 20, 20 - xxlor 21+32, 21, 21 - xxlor 22+32, 22, 22 +__PreLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x vcipher 15 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state + lxv 32+1, 0(10) # last round key (v1) + + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash + +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_enc: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 + + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 # ghash here - ppc_aes_gcm_ghash2_4x - - xxlor 27+32, 0, 0 - vaddudm 30, 30, 31 # IV + counter - vmr 29, 30 - vxor 15, 30, 27 # add round key - vaddudm 30, 30, 31 - vxor 16, 30, 27 - vaddudm 30, 30, 31 - vxor 17, 30, 27 - vaddudm 30, 30, 31 - vxor 18, 30, 27 - vaddudm 30, 30, 31 - vxor 19, 30, 27 - vaddudm 30, 30, 31 - vxor 20, 30, 27 - vaddudm 30, 30, 31 - vxor 21, 30, 27 - vaddudm 30, 30, 31 - vxor 22, 30, 27 - - addi 12, 12, -128 + vxor 15, 15, 0 + PPC_GHASH4x 0, 15, 16, 17, 18 + + vxor 19, 19, 0 + PPC_GHASH4x 0, 19, 20, 21, 22 + + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlor 32+27, 0, 0 # restore roundkey 0 + vxor 15, 15, 27 # IV + round key - add round key 0 + vxor 16, 16, 27 + vxor 17, 17, 27 + vxor 18, 18, 27 + vxor 19, 19, 27 + vxor 20, 20, 27 + vxor 21, 21, 27 + vxor 22, 22, 27 + + addi 5, 5, -128 addi 11, 11, 128 - bdnz Loop_8x_block_dec - - vmr 30, 29 - stxvb16x 30+32, 0, 7 # update IV - -Loop_last_block_dec: - cmpdi 12, 0 - beq aes_gcm_out - - # loop last few blocks - li 10, 16 - divdu 10, 12, 10 - - mtctr 10 - - lwz 10, 240(6) - - cmpdi 12, 16 - blt Final_block_dec - -Next_rem_block_dec: - lxvb16x 15, 0, 14 # load block - - Loop_aes_middle_1x - - xxlor 23+32, 10, 10 + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 +__LastLoop_aes_state: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x vcipher 15 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state + lxv 32+1, 0(10) # last round key (v1) - cmpdi 10, 10 - beq Do_next_1x_dec + addi 12, 12, -1 + cmpdi 12, 0 + bne __Loop_8x_block_enc + +__Finish_ghash: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 + + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 + + vxor 15, 15, 0 + PPC_GHASH4x 0, 15, 16, 17, 18 + + vxor 19, 19, 0 + PPC_GHASH4x 0, 19, 20, 21, 22 + + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+30, 0, 7 # update IV + stxvb16x 32+0, 0, 8 # update Xi + + addi 5, 5, -128 + addi 11, 11, 128 - # 192 bits - xxlor 24+32, 11, 11 + # + # Done 8x blocks + # - vcipher 15, 15, 23 - vcipher 15, 15, 24 + cmpdi 5, 0 + beq aes_gcm_out - xxlor 23+32, 12, 12 +__Process_more_enc: + li 24, 1 # encrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out - cmpdi 10, 12 - beq Do_next_1x_dec + bl __Process_partial + cmpdi 5, 0 + beq aes_gcm_out +__Do_combine_enc: + bl __Combine_partial + cmpdi 5, 0 + bgt __Process_encrypt + b aes_gcm_out - # 256 bits - xxlor 24+32, 13, 13 +SYM_FUNC_END(aes_p10_gcm_encrypt) - vcipher 15, 15, 23 - vcipher 15, 15, 24 +################################################################################ +# aes_p10_gcm_decrypt (const void *inp, void *out, size_t len, +# const char *rk, unsigned char iv[16], void *Xip); +# 8x Decrypt +# +################################################################################ +SYM_FUNC_START(aes_p10_gcm_decrypt) - xxlor 23+32, 14, 14 + cmpdi 5, 0 + ble __Invalid_msg_len - cmpdi 10, 14 - beq Do_next_1x_dec + SAVE_REGS + LOAD_HASH_TABLE -Do_next_1x_dec: - vcipherlast 15, 15, 23 + # initialize ICB: GHASH( IV ), IV - r7 + lxvb16x 30+32, 0, 7 # load IV - v30 - xxlxor 47, 47, 15 - stxvb16x 47, 0, 9 # store output - addi 14, 14, 16 - addi 9, 9, 16 + mr 14, 3 + mr 9, 4 - xxlor 28+32, 15, 15 - #vmr 28, 15 - ppc_update_hash_1x + # counter 1 + vxor 31, 31, 31 + vspltisb 22, 1 + vsldoi 31, 31, 22,1 # counter 1 - addi 12, 12, -16 - addi 11, 11, 16 - xxlor 19+32, 0, 0 - vaddudm 30, 30, 31 # IV + counter - vxor 15, 30, 19 # add round key + addis 11, 2, permx@toc@ha + addi 11, 11, permx@toc@l + lxv 10, 0(11) # vs10: vpermxor vector + li 11, 0 + + # load 9 round keys to VSR + lxv 0, 0(6) # round key 0 + lxv 1, 16(6) # round key 1 + lxv 2, 32(6) # round key 2 + lxv 3, 48(6) # round key 3 + lxv 4, 64(6) # round key 4 + lxv 5, 80(6) # round key 5 + lxv 6, 96(6) # round key 6 + lxv 7, 112(6) # round key 7 + lxv 8, 128(6) # round key 8 - bdnz Next_rem_block_dec + # load rounds - 10 (128), 12 (192), 14 (256) + lwz 23, 240(6) # n rounds + li 24, 0 # decrypt - li 15, 0 - std 15, 56(7) # clear partial? - stxvb16x 30+32, 0, 7 # update IV +__Process_decrypt: + # + # Process different blocks + # + ld 12, 56(7) cmpdi 12, 0 - beq aes_gcm_out - -Final_block_dec: - lwz 10, 240(6) - Loop_aes_middle_1x - - xxlor 23+32, 10, 10 - - cmpdi 10, 10 - beq Do_final_1x_dec + bgt __Do_combine_dec + cmpdi 5, 128 + blt __Process_more_dec + +# +# Process 8x AES/GCM blocks +# +__Process_8x_dec: + # 8x blcoks + li 10, 128 + divdu 12, 5, 10 # n 128 bytes-blocks - # 192 bits - xxlor 24+32, 11, 11 + addi 12, 12, -1 # loop - 1 - vcipher 15, 15, 23 - vcipher 15, 15, 24 + vmr 15, 30 # first state: IV + vadduwm 16, 15, 31 # state + counter + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state - xxlor 23+32, 12, 12 + # vxor state, state, w # addroundkey + xxlor 32+29, 0, 0 + vxor 15, 15, 29 # IV + round key - add round key 0 + vxor 16, 16, 29 + vxor 17, 17, 29 + vxor 18, 18, 29 + vxor 19, 19, 29 + vxor 20, 20, 29 + vxor 21, 21, 29 + vxor 22, 22, 29 - cmpdi 10, 12 - beq Do_final_1x_dec + li 15, 16 + li 16, 32 + li 17, 48 + li 18, 64 + li 19, 80 + li 20, 96 + li 21, 112 - # 256 bits - xxlor 24+32, 13, 13 + # + # Pre-compute first 8 AES state and leave 1/3/5 more rounds + # for the loop. + # + addi 22, 23, -9 # process 8 keys + mtctr 22 # AES key loop + addi 10, 6, 144 - vcipher 15, 15, 23 - vcipher 15, 15, 24 + LOOP_8AES_STATE # process 8 AES keys - xxlor 23+32, 14, 14 +__PreLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x vcipher 15 1 + addi 10, 10, 16 + bdnz __PreLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) + + cmpdi 12, 0 # Only one loop (8 block) + beq __Finish_ghash_dec + +# +# Loop 8x blocks and compute ghash +# +__Loop_8x_block_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 + + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + + addi 9, 9, 128 + + vmr 15, 23 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 - cmpdi 10, 14 - beq Do_final_1x_dec + # ghash here + vxor 15, 15, 0 + PPC_GHASH4x 0, 15, 16, 17, 18 + + vxor 19, 19, 0 + PPC_GHASH4x 0, 19, 20, 21, 22 + + xxlor 32+15, 9, 9 # last state + vadduwm 15, 15, 31 # state + counter + vadduwm 16, 15, 31 + vadduwm 17, 16, 31 + vadduwm 18, 17, 31 + vadduwm 19, 18, 31 + vadduwm 20, 19, 31 + vadduwm 21, 20, 31 + vadduwm 22, 21, 31 + xxlor 9, 32+22, 32+22 # save last state + + xxlor 32+27, 0, 0 # restore roundkey 0 + vxor 15, 15, 27 # IV + round key - add round key 0 + vxor 16, 16, 27 + vxor 17, 17, 27 + vxor 18, 18, 27 + vxor 19, 19, 27 + vxor 20, 20, 27 + vxor 21, 21, 27 + vxor 22, 22, 27 + + addi 5, 5, -128 + addi 11, 11, 128 -Do_final_1x_dec: - vcipherlast 15, 15, 23 + LOOP_8AES_STATE # process 8 AES keys + mtctr 22 # AES key loop + addi 10, 6, 144 +__LastLoop_aes_state_dec: + lxv 32+1, 0(10) # round key + AES_CIPHER_8x vcipher 15 1 + addi 10, 10, 16 + bdnz __LastLoop_aes_state_dec + lxv 32+1, 0(10) # last round key (v1) - # check partial block - li 21, 1 # decrypt - ld 15, 56(7) # partial? - cmpdi 15, 0 - beq Normal_block_dec - bl Do_partial_block + addi 12, 12, -1 cmpdi 12, 0 - ble aes_gcm_out - - b Continue_partial_check_dec + bne __Loop_8x_block_dec + +__Finish_ghash_dec: + vcipherlast 15, 15, 1 + vcipherlast 16, 16, 1 + vcipherlast 17, 17, 1 + vcipherlast 18, 18, 1 + vcipherlast 19, 19, 1 + vcipherlast 20, 20, 1 + vcipherlast 21, 21, 1 + vcipherlast 22, 22, 1 + + lxvb16x 32+23, 0, 14 # load block + lxvb16x 32+24, 15, 14 # load block + lxvb16x 32+25, 16, 14 # load block + lxvb16x 32+26, 17, 14 # load block + lxvb16x 32+27, 18, 14 # load block + lxvb16x 32+28, 19, 14 # load block + lxvb16x 32+29, 20, 14 # load block + lxvb16x 32+30, 21, 14 # load block + addi 14, 14, 128 + + vxor 15, 15, 23 + vxor 16, 16, 24 + vxor 17, 17, 25 + vxor 18, 18, 26 + vxor 19, 19, 27 + vxor 20, 20, 28 + vxor 21, 21, 29 + vxor 22, 22, 30 + + stxvb16x 47, 0, 9 # store output + stxvb16x 48, 15, 9 # store output + stxvb16x 49, 16, 9 # store output + stxvb16x 50, 17, 9 # store output + stxvb16x 51, 18, 9 # store output + stxvb16x 52, 19, 9 # store output + stxvb16x 53, 20, 9 # store output + stxvb16x 54, 21, 9 # store output + addi 9, 9, 128 + + #vmr 15, 23 + vxor 15, 23, 0 + vmr 16, 24 + vmr 17, 25 + vmr 18, 26 + vmr 19, 27 + vmr 20, 28 + vmr 21, 29 + vmr 22, 30 + + #vxor 15, 15, 0 + PPC_GHASH4x 0, 15, 16, 17, 18 + + vxor 19, 19, 0 + PPC_GHASH4x 0, 19, 20, 21, 22 + + xxlor 30+32, 9, 9 # last ctr + vadduwm 30, 30, 31 # increase ctr + stxvb16x 32+30, 0, 7 # update IV + stxvb16x 32+0, 0, 8 # update Xi + + addi 5, 5, -128 + addi 11, 11, 128 -Normal_block_dec: - lxvb16x 15, 0, 14 # load last block - xxlxor 47, 47, 15 + # + # Done 8x blocks + # - # create partial block mask - li 15, 16 - sub 15, 15, 12 # index to the mask + cmpdi 5, 0 + beq aes_gcm_out - vspltisb 16, -1 # first 16 bytes - 0xffff...ff - vspltisb 17, 0 # second 16 bytes - 0x0000...00 - li 10, 192 - stvx 16, 10, 1 - addi 10, 10, 16 - stvx 17, 10, 1 +__Process_more_dec: + li 24, 0 # decrypt + bl aes_gcm_crypt_1x + cmpdi 5, 0 + beq aes_gcm_out - addi 10, 1, 192 - lxvb16x 16, 15, 10 # load partial block mask - xxland 47, 47, 16 + bl __Process_partial + cmpdi 5, 0 + beq aes_gcm_out +__Do_combine_dec: + bl __Combine_partial + cmpdi 5, 0 + bgt __Process_decrypt + b aes_gcm_out +SYM_FUNC_END(aes_p10_gcm_decrypt) - xxland 32+28, 15, 16 - #vmr 28, 15 - ppc_update_hash_1x +SYM_FUNC_START_LOCAL(aes_gcm_out) - # * should store only the remaining bytes. - bl Write_partial_block + mr 3, 11 # return count - stxvb16x 30+32, 0, 7 # update IV - std 12, 56(7) # update partial? - li 16, 16 + RESTORE_REGS + blr - stxvb16x 32, 0, 8 # write out Xi - stxvb16x 32, 16, 8 # write out Xi - b aes_gcm_out +__Invalid_msg_len: + li 3, 0 + blr +SYM_FUNC_END(aes_gcm_out) + +SYM_DATA_START_LOCAL(PERMX) +.align 4 +# for vector permute and xor +permx: +.long 0x4c5d6e7f, 0x08192a3b, 0xc4d5e6f7, 0x8091a2b3 +SYM_DATA_END(permx) diff --git a/arch/x86/crypto/Kconfig b/arch/x86/crypto/Kconfig index 7b1bebed879d..3d2e38ba5240 100644 --- a/arch/x86/crypto/Kconfig +++ b/arch/x86/crypto/Kconfig @@ -363,7 +363,7 @@ config CRYPTO_CHACHA20_X86_64 - AVX-512VL (Advanced Vector Extensions-512VL) config CRYPTO_AEGIS128_AESNI_SSE2 - tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE2)" + tristate "AEAD ciphers: AEGIS-128 (AES-NI/SSE4.1)" depends on X86 && 64BIT select CRYPTO_AEAD select CRYPTO_SIMD @@ -372,7 +372,7 @@ config CRYPTO_AEGIS128_AESNI_SSE2 Architecture: x86_64 using: - AES-NI (AES New Instructions) - - SSE2 (Streaming SIMD Extensions 2) + - SSE4.1 (Streaming SIMD Extensions 4.1) config CRYPTO_NHPOLY1305_SSE2 tristate "Hash functions: NHPoly1305 (SSE2)" diff --git a/arch/x86/crypto/aegis128-aesni-asm.S b/arch/x86/crypto/aegis128-aesni-asm.S index ad7f4c891625..7294dc0ee7ba 100644 --- a/arch/x86/crypto/aegis128-aesni-asm.S +++ b/arch/x86/crypto/aegis128-aesni-asm.S @@ -1,14 +1,13 @@ /* SPDX-License-Identifier: GPL-2.0-only */ /* - * AES-NI + SSE2 implementation of AEGIS-128 + * AES-NI + SSE4.1 implementation of AEGIS-128 * * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com> * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. + * Copyright 2024 Google LLC */ #include <linux/linkage.h> -#include <linux/cfi_types.h> -#include <asm/frame.h> #define STATE0 %xmm0 #define STATE1 %xmm1 @@ -20,11 +19,6 @@ #define T0 %xmm6 #define T1 %xmm7 -#define STATEP %rdi -#define LEN %rsi -#define SRC %rdx -#define DST %rcx - .section .rodata.cst16.aegis128_const, "aM", @progbits, 32 .align 16 .Laegis128_const_0: @@ -34,11 +28,11 @@ .byte 0xdb, 0x3d, 0x18, 0x55, 0x6d, 0xc2, 0x2f, 0xf1 .byte 0x20, 0x11, 0x31, 0x42, 0x73, 0xb5, 0x28, 0xdd -.section .rodata.cst16.aegis128_counter, "aM", @progbits, 16 -.align 16 -.Laegis128_counter: - .byte 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07 - .byte 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f +.section .rodata.cst32.zeropad_mask, "aM", @progbits, 32 +.align 32 +.Lzeropad_mask: + .octa 0xffffffffffffffffffffffffffffffff + .octa 0 .text @@ -61,140 +55,102 @@ .endm /* - * __load_partial: internal ABI - * input: - * LEN - bytes - * SRC - src - * output: - * MSG - message block - * changed: - * T0 - * %r8 - * %r9 + * Load 1 <= LEN (%ecx) <= 15 bytes from the pointer SRC into the xmm register + * MSG and zeroize any remaining bytes. Clobbers %rax, %rcx, and %r8. */ -SYM_FUNC_START_LOCAL(__load_partial) - xor %r9d, %r9d - pxor MSG, MSG - - mov LEN, %r8 - and $0x1, %r8 - jz .Lld_partial_1 - - mov LEN, %r8 - and $0x1E, %r8 - add SRC, %r8 - mov (%r8), %r9b - -.Lld_partial_1: - mov LEN, %r8 - and $0x2, %r8 - jz .Lld_partial_2 - - mov LEN, %r8 - and $0x1C, %r8 - add SRC, %r8 - shl $0x10, %r9 - mov (%r8), %r9w - -.Lld_partial_2: - mov LEN, %r8 - and $0x4, %r8 - jz .Lld_partial_4 - - mov LEN, %r8 - and $0x18, %r8 - add SRC, %r8 - shl $32, %r9 - mov (%r8), %r8d - xor %r8, %r9 - -.Lld_partial_4: - movq %r9, MSG - - mov LEN, %r8 - and $0x8, %r8 - jz .Lld_partial_8 - - mov LEN, %r8 - and $0x10, %r8 - add SRC, %r8 - pslldq $8, MSG - movq (%r8), T0 - pxor T0, MSG - -.Lld_partial_8: - RET -SYM_FUNC_END(__load_partial) +.macro load_partial + sub $8, %ecx /* LEN - 8 */ + jle .Lle8\@ + + /* Load 9 <= LEN <= 15 bytes: */ + movq (SRC), MSG /* Load first 8 bytes */ + mov (SRC, %rcx), %rax /* Load last 8 bytes */ + neg %ecx + shl $3, %ecx + shr %cl, %rax /* Discard overlapping bytes */ + pinsrq $1, %rax, MSG + jmp .Ldone\@ + +.Lle8\@: + add $4, %ecx /* LEN - 4 */ + jl .Llt4\@ + + /* Load 4 <= LEN <= 8 bytes: */ + mov (SRC), %eax /* Load first 4 bytes */ + mov (SRC, %rcx), %r8d /* Load last 4 bytes */ + jmp .Lcombine\@ + +.Llt4\@: + /* Load 1 <= LEN <= 3 bytes: */ + add $2, %ecx /* LEN - 2 */ + movzbl (SRC), %eax /* Load first byte */ + jl .Lmovq\@ + movzwl (SRC, %rcx), %r8d /* Load last 2 bytes */ +.Lcombine\@: + shl $3, %ecx + shl %cl, %r8 + or %r8, %rax /* Combine the two parts */ +.Lmovq\@: + movq %rax, MSG +.Ldone\@: +.endm /* - * __store_partial: internal ABI - * input: - * LEN - bytes - * DST - dst - * output: - * T0 - message block - * changed: - * %r8 - * %r9 - * %r10 + * Store 1 <= LEN (%ecx) <= 15 bytes from the xmm register \msg to the pointer + * DST. Clobbers %rax, %rcx, and %r8. */ -SYM_FUNC_START_LOCAL(__store_partial) - mov LEN, %r8 - mov DST, %r9 - - movq T0, %r10 - - cmp $8, %r8 - jl .Lst_partial_8 - - mov %r10, (%r9) - psrldq $8, T0 - movq T0, %r10 - - sub $8, %r8 - add $8, %r9 - -.Lst_partial_8: - cmp $4, %r8 - jl .Lst_partial_4 - - mov %r10d, (%r9) - shr $32, %r10 - - sub $4, %r8 - add $4, %r9 - -.Lst_partial_4: - cmp $2, %r8 - jl .Lst_partial_2 - - mov %r10w, (%r9) - shr $0x10, %r10 - - sub $2, %r8 - add $2, %r9 - -.Lst_partial_2: - cmp $1, %r8 - jl .Lst_partial_1 - - mov %r10b, (%r9) - -.Lst_partial_1: - RET -SYM_FUNC_END(__store_partial) +.macro store_partial msg + sub $8, %ecx /* LEN - 8 */ + jl .Llt8\@ + + /* Store 8 <= LEN <= 15 bytes: */ + pextrq $1, \msg, %rax + mov %ecx, %r8d + shl $3, %ecx + ror %cl, %rax + mov %rax, (DST, %r8) /* Store last LEN - 8 bytes */ + movq \msg, (DST) /* Store first 8 bytes */ + jmp .Ldone\@ + +.Llt8\@: + add $4, %ecx /* LEN - 4 */ + jl .Llt4\@ + + /* Store 4 <= LEN <= 7 bytes: */ + pextrd $1, \msg, %eax + mov %ecx, %r8d + shl $3, %ecx + ror %cl, %eax + mov %eax, (DST, %r8) /* Store last LEN - 4 bytes */ + movd \msg, (DST) /* Store first 4 bytes */ + jmp .Ldone\@ + +.Llt4\@: + /* Store 1 <= LEN <= 3 bytes: */ + pextrb $0, \msg, 0(DST) + cmp $-2, %ecx /* LEN - 4 == -2, i.e. LEN == 2? */ + jl .Ldone\@ + pextrb $1, \msg, 1(DST) + je .Ldone\@ + pextrb $2, \msg, 2(DST) +.Ldone\@: +.endm /* - * void crypto_aegis128_aesni_init(void *state, const void *key, const void *iv); + * void aegis128_aesni_init(struct aegis_state *state, + * const struct aegis_block *key, + * const u8 iv[AEGIS128_NONCE_SIZE]); */ -SYM_FUNC_START(crypto_aegis128_aesni_init) - FRAME_BEGIN +SYM_FUNC_START(aegis128_aesni_init) + .set STATEP, %rdi + .set KEYP, %rsi + .set IVP, %rdx /* load IV: */ - movdqu (%rdx), T1 + movdqu (IVP), T1 /* load key: */ - movdqa (%rsi), KEY + movdqa (KEYP), KEY pxor KEY, T1 movdqa T1, STATE0 movdqa KEY, STATE3 @@ -224,20 +180,22 @@ SYM_FUNC_START(crypto_aegis128_aesni_init) movdqu STATE2, 0x20(STATEP) movdqu STATE3, 0x30(STATEP) movdqu STATE4, 0x40(STATEP) - - FRAME_END RET -SYM_FUNC_END(crypto_aegis128_aesni_init) +SYM_FUNC_END(aegis128_aesni_init) /* - * void crypto_aegis128_aesni_ad(void *state, unsigned int length, - * const void *data); + * void aegis128_aesni_ad(struct aegis_state *state, const u8 *data, + * unsigned int len); + * + * len must be a multiple of 16. */ -SYM_FUNC_START(crypto_aegis128_aesni_ad) - FRAME_BEGIN +SYM_FUNC_START(aegis128_aesni_ad) + .set STATEP, %rdi + .set SRC, %rsi + .set LEN, %edx - cmp $0x10, LEN - jb .Lad_out + test LEN, LEN + jz .Lad_out /* load the state: */ movdqu 0x00(STATEP), STATE0 @@ -246,89 +204,40 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad) movdqu 0x30(STATEP), STATE3 movdqu 0x40(STATEP), STATE4 - mov SRC, %r8 - and $0xF, %r8 - jnz .Lad_u_loop - -.align 8 -.Lad_a_loop: - movdqa 0x00(SRC), MSG - aegis128_update - pxor MSG, STATE4 - sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_1 - - movdqa 0x10(SRC), MSG - aegis128_update - pxor MSG, STATE3 - sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_2 - - movdqa 0x20(SRC), MSG - aegis128_update - pxor MSG, STATE2 - sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_3 - - movdqa 0x30(SRC), MSG - aegis128_update - pxor MSG, STATE1 - sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_4 - - movdqa 0x40(SRC), MSG - aegis128_update - pxor MSG, STATE0 - sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_0 - - add $0x50, SRC - jmp .Lad_a_loop - .align 8 -.Lad_u_loop: +.Lad_loop: movdqu 0x00(SRC), MSG aegis128_update pxor MSG, STATE4 sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_1 + jz .Lad_out_1 movdqu 0x10(SRC), MSG aegis128_update pxor MSG, STATE3 sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_2 + jz .Lad_out_2 movdqu 0x20(SRC), MSG aegis128_update pxor MSG, STATE2 sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_3 + jz .Lad_out_3 movdqu 0x30(SRC), MSG aegis128_update pxor MSG, STATE1 sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_4 + jz .Lad_out_4 movdqu 0x40(SRC), MSG aegis128_update pxor MSG, STATE0 sub $0x10, LEN - cmp $0x10, LEN - jl .Lad_out_0 + jz .Lad_out_0 add $0x50, SRC - jmp .Lad_u_loop + jmp .Lad_loop /* store the state: */ .Lad_out_0: @@ -337,7 +246,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad) movdqu STATE2, 0x20(STATEP) movdqu STATE3, 0x30(STATEP) movdqu STATE4, 0x40(STATEP) - FRAME_END RET .Lad_out_1: @@ -346,7 +254,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad) movdqu STATE1, 0x20(STATEP) movdqu STATE2, 0x30(STATEP) movdqu STATE3, 0x40(STATEP) - FRAME_END RET .Lad_out_2: @@ -355,7 +262,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad) movdqu STATE0, 0x20(STATEP) movdqu STATE1, 0x30(STATEP) movdqu STATE2, 0x40(STATEP) - FRAME_END RET .Lad_out_3: @@ -364,7 +270,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad) movdqu STATE4, 0x20(STATEP) movdqu STATE0, 0x30(STATEP) movdqu STATE1, 0x40(STATEP) - FRAME_END RET .Lad_out_4: @@ -373,41 +278,38 @@ SYM_FUNC_START(crypto_aegis128_aesni_ad) movdqu STATE3, 0x20(STATEP) movdqu STATE4, 0x30(STATEP) movdqu STATE0, 0x40(STATEP) - FRAME_END - RET - .Lad_out: - FRAME_END RET -SYM_FUNC_END(crypto_aegis128_aesni_ad) +SYM_FUNC_END(aegis128_aesni_ad) -.macro encrypt_block a s0 s1 s2 s3 s4 i - movdq\a (\i * 0x10)(SRC), MSG +.macro encrypt_block s0 s1 s2 s3 s4 i + movdqu (\i * 0x10)(SRC), MSG movdqa MSG, T0 pxor \s1, T0 pxor \s4, T0 movdqa \s2, T1 pand \s3, T1 pxor T1, T0 - movdq\a T0, (\i * 0x10)(DST) + movdqu T0, (\i * 0x10)(DST) aegis128_update pxor MSG, \s4 sub $0x10, LEN - cmp $0x10, LEN - jl .Lenc_out_\i + jz .Lenc_out_\i .endm /* - * void crypto_aegis128_aesni_enc(void *state, unsigned int length, - * const void *src, void *dst); + * void aegis128_aesni_enc(struct aegis_state *state, const u8 *src, u8 *dst, + * unsigned int len); + * + * len must be nonzero and a multiple of 16. */ -SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc) - FRAME_BEGIN - - cmp $0x10, LEN - jb .Lenc_out +SYM_FUNC_START(aegis128_aesni_enc) + .set STATEP, %rdi + .set SRC, %rsi + .set DST, %rdx + .set LEN, %ecx /* load the state: */ movdqu 0x00(STATEP), STATE0 @@ -416,34 +318,17 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc) movdqu 0x30(STATEP), STATE3 movdqu 0x40(STATEP), STATE4 - mov SRC, %r8 - or DST, %r8 - and $0xF, %r8 - jnz .Lenc_u_loop - .align 8 -.Lenc_a_loop: - encrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0 - encrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1 - encrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2 - encrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3 - encrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4 +.Lenc_loop: + encrypt_block STATE0 STATE1 STATE2 STATE3 STATE4 0 + encrypt_block STATE4 STATE0 STATE1 STATE2 STATE3 1 + encrypt_block STATE3 STATE4 STATE0 STATE1 STATE2 2 + encrypt_block STATE2 STATE3 STATE4 STATE0 STATE1 3 + encrypt_block STATE1 STATE2 STATE3 STATE4 STATE0 4 add $0x50, SRC add $0x50, DST - jmp .Lenc_a_loop - -.align 8 -.Lenc_u_loop: - encrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0 - encrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1 - encrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2 - encrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3 - encrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4 - - add $0x50, SRC - add $0x50, DST - jmp .Lenc_u_loop + jmp .Lenc_loop /* store the state: */ .Lenc_out_0: @@ -452,7 +337,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc) movdqu STATE1, 0x20(STATEP) movdqu STATE2, 0x30(STATEP) movdqu STATE3, 0x40(STATEP) - FRAME_END RET .Lenc_out_1: @@ -461,7 +345,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc) movdqu STATE0, 0x20(STATEP) movdqu STATE1, 0x30(STATEP) movdqu STATE2, 0x40(STATEP) - FRAME_END RET .Lenc_out_2: @@ -470,7 +353,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc) movdqu STATE4, 0x20(STATEP) movdqu STATE0, 0x30(STATEP) movdqu STATE1, 0x40(STATEP) - FRAME_END RET .Lenc_out_3: @@ -479,7 +361,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc) movdqu STATE3, 0x20(STATEP) movdqu STATE4, 0x30(STATEP) movdqu STATE0, 0x40(STATEP) - FRAME_END RET .Lenc_out_4: @@ -488,20 +369,19 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc) movdqu STATE2, 0x20(STATEP) movdqu STATE3, 0x30(STATEP) movdqu STATE4, 0x40(STATEP) - FRAME_END - RET - .Lenc_out: - FRAME_END RET -SYM_FUNC_END(crypto_aegis128_aesni_enc) +SYM_FUNC_END(aegis128_aesni_enc) /* - * void crypto_aegis128_aesni_enc_tail(void *state, unsigned int length, - * const void *src, void *dst); + * void aegis128_aesni_enc_tail(struct aegis_state *state, const u8 *src, + * u8 *dst, unsigned int len); */ -SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail) - FRAME_BEGIN +SYM_FUNC_START(aegis128_aesni_enc_tail) + .set STATEP, %rdi + .set SRC, %rsi + .set DST, %rdx + .set LEN, %ecx /* {load,store}_partial rely on this being %ecx */ /* load the state: */ movdqu 0x00(STATEP), STATE0 @@ -511,7 +391,8 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail) movdqu 0x40(STATEP), STATE4 /* encrypt message: */ - call __load_partial + mov LEN, %r9d + load_partial movdqa MSG, T0 pxor STATE1, T0 @@ -520,7 +401,8 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail) pand STATE3, T1 pxor T1, T0 - call __store_partial + mov %r9d, LEN + store_partial T0 aegis128_update pxor MSG, STATE4 @@ -531,37 +413,36 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_enc_tail) movdqu STATE1, 0x20(STATEP) movdqu STATE2, 0x30(STATEP) movdqu STATE3, 0x40(STATEP) - - FRAME_END RET -SYM_FUNC_END(crypto_aegis128_aesni_enc_tail) +SYM_FUNC_END(aegis128_aesni_enc_tail) -.macro decrypt_block a s0 s1 s2 s3 s4 i - movdq\a (\i * 0x10)(SRC), MSG +.macro decrypt_block s0 s1 s2 s3 s4 i + movdqu (\i * 0x10)(SRC), MSG pxor \s1, MSG pxor \s4, MSG movdqa \s2, T1 pand \s3, T1 pxor T1, MSG - movdq\a MSG, (\i * 0x10)(DST) + movdqu MSG, (\i * 0x10)(DST) aegis128_update pxor MSG, \s4 sub $0x10, LEN - cmp $0x10, LEN - jl .Ldec_out_\i + jz .Ldec_out_\i .endm /* - * void crypto_aegis128_aesni_dec(void *state, unsigned int length, - * const void *src, void *dst); + * void aegis128_aesni_dec(struct aegis_state *state, const u8 *src, u8 *dst, + * unsigned int len); + * + * len must be nonzero and a multiple of 16. */ -SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec) - FRAME_BEGIN - - cmp $0x10, LEN - jb .Ldec_out +SYM_FUNC_START(aegis128_aesni_dec) + .set STATEP, %rdi + .set SRC, %rsi + .set DST, %rdx + .set LEN, %ecx /* load the state: */ movdqu 0x00(STATEP), STATE0 @@ -570,34 +451,17 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec) movdqu 0x30(STATEP), STATE3 movdqu 0x40(STATEP), STATE4 - mov SRC, %r8 - or DST, %r8 - and $0xF, %r8 - jnz .Ldec_u_loop - .align 8 -.Ldec_a_loop: - decrypt_block a STATE0 STATE1 STATE2 STATE3 STATE4 0 - decrypt_block a STATE4 STATE0 STATE1 STATE2 STATE3 1 - decrypt_block a STATE3 STATE4 STATE0 STATE1 STATE2 2 - decrypt_block a STATE2 STATE3 STATE4 STATE0 STATE1 3 - decrypt_block a STATE1 STATE2 STATE3 STATE4 STATE0 4 +.Ldec_loop: + decrypt_block STATE0 STATE1 STATE2 STATE3 STATE4 0 + decrypt_block STATE4 STATE0 STATE1 STATE2 STATE3 1 + decrypt_block STATE3 STATE4 STATE0 STATE1 STATE2 2 + decrypt_block STATE2 STATE3 STATE4 STATE0 STATE1 3 + decrypt_block STATE1 STATE2 STATE3 STATE4 STATE0 4 add $0x50, SRC add $0x50, DST - jmp .Ldec_a_loop - -.align 8 -.Ldec_u_loop: - decrypt_block u STATE0 STATE1 STATE2 STATE3 STATE4 0 - decrypt_block u STATE4 STATE0 STATE1 STATE2 STATE3 1 - decrypt_block u STATE3 STATE4 STATE0 STATE1 STATE2 2 - decrypt_block u STATE2 STATE3 STATE4 STATE0 STATE1 3 - decrypt_block u STATE1 STATE2 STATE3 STATE4 STATE0 4 - - add $0x50, SRC - add $0x50, DST - jmp .Ldec_u_loop + jmp .Ldec_loop /* store the state: */ .Ldec_out_0: @@ -606,7 +470,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec) movdqu STATE1, 0x20(STATEP) movdqu STATE2, 0x30(STATEP) movdqu STATE3, 0x40(STATEP) - FRAME_END RET .Ldec_out_1: @@ -615,7 +478,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec) movdqu STATE0, 0x20(STATEP) movdqu STATE1, 0x30(STATEP) movdqu STATE2, 0x40(STATEP) - FRAME_END RET .Ldec_out_2: @@ -624,7 +486,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec) movdqu STATE4, 0x20(STATEP) movdqu STATE0, 0x30(STATEP) movdqu STATE1, 0x40(STATEP) - FRAME_END RET .Ldec_out_3: @@ -633,7 +494,6 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec) movdqu STATE3, 0x20(STATEP) movdqu STATE4, 0x30(STATEP) movdqu STATE0, 0x40(STATEP) - FRAME_END RET .Ldec_out_4: @@ -642,20 +502,19 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec) movdqu STATE2, 0x20(STATEP) movdqu STATE3, 0x30(STATEP) movdqu STATE4, 0x40(STATEP) - FRAME_END - RET - .Ldec_out: - FRAME_END RET -SYM_FUNC_END(crypto_aegis128_aesni_dec) +SYM_FUNC_END(aegis128_aesni_dec) /* - * void crypto_aegis128_aesni_dec_tail(void *state, unsigned int length, - * const void *src, void *dst); + * void aegis128_aesni_dec_tail(struct aegis_state *state, const u8 *src, + * u8 *dst, unsigned int len); */ -SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail) - FRAME_BEGIN +SYM_FUNC_START(aegis128_aesni_dec_tail) + .set STATEP, %rdi + .set SRC, %rsi + .set DST, %rdx + .set LEN, %ecx /* {load,store}_partial rely on this being %ecx */ /* load the state: */ movdqu 0x00(STATEP), STATE0 @@ -665,7 +524,8 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail) movdqu 0x40(STATEP), STATE4 /* decrypt message: */ - call __load_partial + mov LEN, %r9d + load_partial pxor STATE1, MSG pxor STATE4, MSG @@ -673,17 +533,13 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail) pand STATE3, T1 pxor T1, MSG - movdqa MSG, T0 - call __store_partial + mov %r9d, LEN + store_partial MSG /* mask with byte count: */ - movq LEN, T0 - punpcklbw T0, T0 - punpcklbw T0, T0 - punpcklbw T0, T0 - punpcklbw T0, T0 - movdqa .Laegis128_counter(%rip), T1 - pcmpgtb T1, T0 + lea .Lzeropad_mask+16(%rip), %rax + sub %r9, %rax + movdqu (%rax), T0 pand T0, MSG aegis128_update @@ -695,17 +551,19 @@ SYM_TYPED_FUNC_START(crypto_aegis128_aesni_dec_tail) movdqu STATE1, 0x20(STATEP) movdqu STATE2, 0x30(STATEP) movdqu STATE3, 0x40(STATEP) - - FRAME_END RET -SYM_FUNC_END(crypto_aegis128_aesni_dec_tail) +SYM_FUNC_END(aegis128_aesni_dec_tail) /* - * void crypto_aegis128_aesni_final(void *state, void *tag_xor, - * u64 assoclen, u64 cryptlen); + * void aegis128_aesni_final(struct aegis_state *state, + * struct aegis_block *tag_xor, + * unsigned int assoclen, unsigned int cryptlen); */ -SYM_FUNC_START(crypto_aegis128_aesni_final) - FRAME_BEGIN +SYM_FUNC_START(aegis128_aesni_final) + .set STATEP, %rdi + .set TAG_XOR, %rsi + .set ASSOCLEN, %edx + .set CRYPTLEN, %ecx /* load the state: */ movdqu 0x00(STATEP), STATE0 @@ -715,10 +573,8 @@ SYM_FUNC_START(crypto_aegis128_aesni_final) movdqu 0x40(STATEP), STATE4 /* prepare length block: */ - movq %rdx, MSG - movq %rcx, T0 - pslldq $8, T0 - pxor T0, MSG + movd ASSOCLEN, MSG + pinsrd $2, CRYPTLEN, MSG psllq $3, MSG /* multiply by 8 (to get bit count) */ pxor STATE3, MSG @@ -733,7 +589,7 @@ SYM_FUNC_START(crypto_aegis128_aesni_final) aegis128_update; pxor MSG, STATE3 /* xor tag: */ - movdqu (%rsi), MSG + movdqu (TAG_XOR), MSG pxor STATE0, MSG pxor STATE1, MSG @@ -741,8 +597,6 @@ SYM_FUNC_START(crypto_aegis128_aesni_final) pxor STATE3, MSG pxor STATE4, MSG - movdqu MSG, (%rsi) - - FRAME_END + movdqu MSG, (TAG_XOR) RET -SYM_FUNC_END(crypto_aegis128_aesni_final) +SYM_FUNC_END(aegis128_aesni_final) diff --git a/arch/x86/crypto/aegis128-aesni-glue.c b/arch/x86/crypto/aegis128-aesni-glue.c index 4623189000d8..c19d8e3d96a3 100644 --- a/arch/x86/crypto/aegis128-aesni-glue.c +++ b/arch/x86/crypto/aegis128-aesni-glue.c @@ -1,7 +1,7 @@ // SPDX-License-Identifier: GPL-2.0-or-later /* * The AEGIS-128 Authenticated-Encryption Algorithm - * Glue for AES-NI + SSE2 implementation + * Glue for AES-NI + SSE4.1 implementation * * Copyright (c) 2017-2018 Ondrej Mosnacek <omosnacek@gmail.com> * Copyright (C) 2017-2018 Red Hat, Inc. All rights reserved. @@ -23,27 +23,6 @@ #define AEGIS128_MIN_AUTH_SIZE 8 #define AEGIS128_MAX_AUTH_SIZE 16 -asmlinkage void crypto_aegis128_aesni_init(void *state, void *key, void *iv); - -asmlinkage void crypto_aegis128_aesni_ad( - void *state, unsigned int length, const void *data); - -asmlinkage void crypto_aegis128_aesni_enc( - void *state, unsigned int length, const void *src, void *dst); - -asmlinkage void crypto_aegis128_aesni_dec( - void *state, unsigned int length, const void *src, void *dst); - -asmlinkage void crypto_aegis128_aesni_enc_tail( - void *state, unsigned int length, const void *src, void *dst); - -asmlinkage void crypto_aegis128_aesni_dec_tail( - void *state, unsigned int length, const void *src, void *dst); - -asmlinkage void crypto_aegis128_aesni_final( - void *state, void *tag_xor, unsigned int cryptlen, - unsigned int assoclen); - struct aegis_block { u8 bytes[AEGIS128_BLOCK_SIZE] __aligned(AEGIS128_BLOCK_ALIGN); }; @@ -56,15 +35,31 @@ struct aegis_ctx { struct aegis_block key; }; -struct aegis_crypt_ops { - int (*skcipher_walk_init)(struct skcipher_walk *walk, - struct aead_request *req, bool atomic); +asmlinkage void aegis128_aesni_init(struct aegis_state *state, + const struct aegis_block *key, + const u8 iv[AEGIS128_NONCE_SIZE]); - void (*crypt_blocks)(void *state, unsigned int length, const void *src, - void *dst); - void (*crypt_tail)(void *state, unsigned int length, const void *src, - void *dst); -}; +asmlinkage void aegis128_aesni_ad(struct aegis_state *state, const u8 *data, + unsigned int len); + +asmlinkage void aegis128_aesni_enc(struct aegis_state *state, const u8 *src, + u8 *dst, unsigned int len); + +asmlinkage void aegis128_aesni_dec(struct aegis_state *state, const u8 *src, + u8 *dst, unsigned int len); + +asmlinkage void aegis128_aesni_enc_tail(struct aegis_state *state, + const u8 *src, u8 *dst, + unsigned int len); + +asmlinkage void aegis128_aesni_dec_tail(struct aegis_state *state, + const u8 *src, u8 *dst, + unsigned int len); + +asmlinkage void aegis128_aesni_final(struct aegis_state *state, + struct aegis_block *tag_xor, + unsigned int assoclen, + unsigned int cryptlen); static void crypto_aegis128_aesni_process_ad( struct aegis_state *state, struct scatterlist *sg_src, @@ -85,16 +80,15 @@ static void crypto_aegis128_aesni_process_ad( if (pos > 0) { unsigned int fill = AEGIS128_BLOCK_SIZE - pos; memcpy(buf.bytes + pos, src, fill); - crypto_aegis128_aesni_ad(state, - AEGIS128_BLOCK_SIZE, - buf.bytes); + aegis128_aesni_ad(state, buf.bytes, + AEGIS128_BLOCK_SIZE); pos = 0; left -= fill; src += fill; } - crypto_aegis128_aesni_ad(state, left, src); - + aegis128_aesni_ad(state, src, + left & ~(AEGIS128_BLOCK_SIZE - 1)); src += left & ~(AEGIS128_BLOCK_SIZE - 1); left &= AEGIS128_BLOCK_SIZE - 1; } @@ -110,24 +104,37 @@ static void crypto_aegis128_aesni_process_ad( if (pos > 0) { memset(buf.bytes + pos, 0, AEGIS128_BLOCK_SIZE - pos); - crypto_aegis128_aesni_ad(state, AEGIS128_BLOCK_SIZE, buf.bytes); + aegis128_aesni_ad(state, buf.bytes, AEGIS128_BLOCK_SIZE); } } -static void crypto_aegis128_aesni_process_crypt( - struct aegis_state *state, struct skcipher_walk *walk, - const struct aegis_crypt_ops *ops) +static __always_inline void +crypto_aegis128_aesni_process_crypt(struct aegis_state *state, + struct skcipher_walk *walk, bool enc) { while (walk->nbytes >= AEGIS128_BLOCK_SIZE) { - ops->crypt_blocks(state, - round_down(walk->nbytes, AEGIS128_BLOCK_SIZE), - walk->src.virt.addr, walk->dst.virt.addr); + if (enc) + aegis128_aesni_enc(state, walk->src.virt.addr, + walk->dst.virt.addr, + round_down(walk->nbytes, + AEGIS128_BLOCK_SIZE)); + else + aegis128_aesni_dec(state, walk->src.virt.addr, + walk->dst.virt.addr, + round_down(walk->nbytes, + AEGIS128_BLOCK_SIZE)); skcipher_walk_done(walk, walk->nbytes % AEGIS128_BLOCK_SIZE); } if (walk->nbytes) { - ops->crypt_tail(state, walk->nbytes, walk->src.virt.addr, - walk->dst.virt.addr); + if (enc) + aegis128_aesni_enc_tail(state, walk->src.virt.addr, + walk->dst.virt.addr, + walk->nbytes); + else + aegis128_aesni_dec_tail(state, walk->src.virt.addr, + walk->dst.virt.addr, + walk->nbytes); skcipher_walk_done(walk, 0); } } @@ -162,42 +169,39 @@ static int crypto_aegis128_aesni_setauthsize(struct crypto_aead *tfm, return 0; } -static void crypto_aegis128_aesni_crypt(struct aead_request *req, - struct aegis_block *tag_xor, - unsigned int cryptlen, - const struct aegis_crypt_ops *ops) +static __always_inline void +crypto_aegis128_aesni_crypt(struct aead_request *req, + struct aegis_block *tag_xor, + unsigned int cryptlen, bool enc) { struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aegis_ctx *ctx = crypto_aegis128_aesni_ctx(tfm); struct skcipher_walk walk; struct aegis_state state; - ops->skcipher_walk_init(&walk, req, true); + if (enc) + skcipher_walk_aead_encrypt(&walk, req, true); + else + skcipher_walk_aead_decrypt(&walk, req, true); kernel_fpu_begin(); - crypto_aegis128_aesni_init(&state, ctx->key.bytes, req->iv); + aegis128_aesni_init(&state, &ctx->key, req->iv); crypto_aegis128_aesni_process_ad(&state, req->src, req->assoclen); - crypto_aegis128_aesni_process_crypt(&state, &walk, ops); - crypto_aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen); + crypto_aegis128_aesni_process_crypt(&state, &walk, enc); + aegis128_aesni_final(&state, tag_xor, req->assoclen, cryptlen); kernel_fpu_end(); } static int crypto_aegis128_aesni_encrypt(struct aead_request *req) { - static const struct aegis_crypt_ops OPS = { - .skcipher_walk_init = skcipher_walk_aead_encrypt, - .crypt_blocks = crypto_aegis128_aesni_enc, - .crypt_tail = crypto_aegis128_aesni_enc_tail, - }; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aegis_block tag = {}; unsigned int authsize = crypto_aead_authsize(tfm); unsigned int cryptlen = req->cryptlen; - crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS); + crypto_aegis128_aesni_crypt(req, &tag, cryptlen, true); scatterwalk_map_and_copy(tag.bytes, req->dst, req->assoclen + cryptlen, authsize, 1); @@ -208,12 +212,6 @@ static int crypto_aegis128_aesni_decrypt(struct aead_request *req) { static const struct aegis_block zeros = {}; - static const struct aegis_crypt_ops OPS = { - .skcipher_walk_init = skcipher_walk_aead_decrypt, - .crypt_blocks = crypto_aegis128_aesni_dec, - .crypt_tail = crypto_aegis128_aesni_dec_tail, - }; - struct crypto_aead *tfm = crypto_aead_reqtfm(req); struct aegis_block tag; unsigned int authsize = crypto_aead_authsize(tfm); @@ -222,27 +220,16 @@ static int crypto_aegis128_aesni_decrypt(struct aead_request *req) scatterwalk_map_and_copy(tag.bytes, req->src, req->assoclen + cryptlen, authsize, 0); - crypto_aegis128_aesni_crypt(req, &tag, cryptlen, &OPS); + crypto_aegis128_aesni_crypt(req, &tag, cryptlen, false); return crypto_memneq(tag.bytes, zeros.bytes, authsize) ? -EBADMSG : 0; } -static int crypto_aegis128_aesni_init_tfm(struct crypto_aead *aead) -{ - return 0; -} - -static void crypto_aegis128_aesni_exit_tfm(struct crypto_aead *aead) -{ -} - static struct aead_alg crypto_aegis128_aesni_alg = { .setkey = crypto_aegis128_aesni_setkey, .setauthsize = crypto_aegis128_aesni_setauthsize, .encrypt = crypto_aegis128_aesni_encrypt, .decrypt = crypto_aegis128_aesni_decrypt, - .init = crypto_aegis128_aesni_init_tfm, - .exit = crypto_aegis128_aesni_exit_tfm, .ivsize = AEGIS128_NONCE_SIZE, .maxauthsize = AEGIS128_MAX_AUTH_SIZE, @@ -267,7 +254,7 @@ static struct simd_aead_alg *simd_alg; static int __init crypto_aegis128_aesni_module_init(void) { - if (!boot_cpu_has(X86_FEATURE_XMM2) || + if (!boot_cpu_has(X86_FEATURE_XMM4_1) || !boot_cpu_has(X86_FEATURE_AES) || !cpu_has_xfeatures(XFEATURE_MASK_SSE, NULL)) return -ENODEV; @@ -286,6 +273,6 @@ module_exit(crypto_aegis128_aesni_module_exit); MODULE_LICENSE("GPL"); MODULE_AUTHOR("Ondrej Mosnacek <omosnacek@gmail.com>"); -MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE2 implementation"); +MODULE_DESCRIPTION("AEGIS-128 AEAD algorithm -- AESNI+SSE4.1 implementation"); MODULE_ALIAS_CRYPTO("aegis128"); MODULE_ALIAS_CRYPTO("aegis128-aesni"); diff --git a/arch/x86/crypto/aesni-intel_glue.c b/arch/x86/crypto/aesni-intel_glue.c index b0dd83555499..fbf43482e1f5 100644 --- a/arch/x86/crypto/aesni-intel_glue.c +++ b/arch/x86/crypto/aesni-intel_glue.c @@ -1747,7 +1747,7 @@ static void __exit aesni_exit(void) unregister_avx_algs(); } -late_initcall(aesni_init); +module_init(aesni_init); module_exit(aesni_exit); MODULE_DESCRIPTION("AES cipher and modes, optimized with AES-NI or VAES instructions"); diff --git a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S index b4e460a87f18..fb95a614249d 100644 --- a/arch/x86/crypto/cast5-avx-x86_64-asm_64.S +++ b/arch/x86/crypto/cast5-avx-x86_64-asm_64.S @@ -487,79 +487,3 @@ SYM_FUNC_START(cast5_cbc_dec_16way) FRAME_END RET; SYM_FUNC_END(cast5_cbc_dec_16way) - -SYM_FUNC_START(cast5_ctr_16way) - /* input: - * %rdi: ctx - * %rsi: dst - * %rdx: src - * %rcx: iv (big endian, 64bit) - */ - FRAME_BEGIN - pushq %r12; - pushq %r15; - - movq %rdi, CTX; - movq %rsi, %r11; - movq %rdx, %r12; - - vpcmpeqd RTMP, RTMP, RTMP; - vpsrldq $8, RTMP, RTMP; /* low: -1, high: 0 */ - - vpcmpeqd RKR, RKR, RKR; - vpaddq RKR, RKR, RKR; /* low: -2, high: -2 */ - vmovdqa .Lbswap_iv_mask(%rip), R1ST; - vmovdqa .Lbswap128_mask(%rip), RKM; - - /* load IV and byteswap */ - vmovq (%rcx), RX; - vpshufb R1ST, RX, RX; - - /* construct IVs */ - vpsubq RTMP, RX, RX; /* le: IV1, IV0 */ - vpshufb RKM, RX, RL1; /* be: IV0, IV1 */ - vpsubq RKR, RX, RX; - vpshufb RKM, RX, RR1; /* be: IV2, IV3 */ - vpsubq RKR, RX, RX; - vpshufb RKM, RX, RL2; /* be: IV4, IV5 */ - vpsubq RKR, RX, RX; - vpshufb RKM, RX, RR2; /* be: IV6, IV7 */ - vpsubq RKR, RX, RX; - vpshufb RKM, RX, RL3; /* be: IV8, IV9 */ - vpsubq RKR, RX, RX; - vpshufb RKM, RX, RR3; /* be: IV10, IV11 */ - vpsubq RKR, RX, RX; - vpshufb RKM, RX, RL4; /* be: IV12, IV13 */ - vpsubq RKR, RX, RX; - vpshufb RKM, RX, RR4; /* be: IV14, IV15 */ - - /* store last IV */ - vpsubq RTMP, RX, RX; /* le: IV16, IV14 */ - vpshufb R1ST, RX, RX; /* be: IV16, IV16 */ - vmovq RX, (%rcx); - - call __cast5_enc_blk16; - - /* dst = src ^ iv */ - vpxor (0*16)(%r12), RR1, RR1; - vpxor (1*16)(%r12), RL1, RL1; - vpxor (2*16)(%r12), RR2, RR2; - vpxor (3*16)(%r12), RL2, RL2; - vpxor (4*16)(%r12), RR3, RR3; - vpxor (5*16)(%r12), RL3, RL3; - vpxor (6*16)(%r12), RR4, RR4; - vpxor (7*16)(%r12), RL4, RL4; - vmovdqu RR1, (0*16)(%r11); - vmovdqu RL1, (1*16)(%r11); - vmovdqu RR2, (2*16)(%r11); - vmovdqu RL2, (3*16)(%r11); - vmovdqu RR3, (4*16)(%r11); - vmovdqu RL3, (5*16)(%r11); - vmovdqu RR4, (6*16)(%r11); - vmovdqu RL4, (7*16)(%r11); - - popq %r15; - popq %r12; - FRAME_END - RET; -SYM_FUNC_END(cast5_ctr_16way) diff --git a/arch/x86/crypto/crc32c-intel_glue.c b/arch/x86/crypto/crc32c-intel_glue.c index feccb5254c7e..52c5d47ef5a1 100644 --- a/arch/x86/crypto/crc32c-intel_glue.c +++ b/arch/x86/crypto/crc32c-intel_glue.c @@ -41,7 +41,7 @@ */ #define CRC32C_PCL_BREAKEVEN 512 -asmlinkage unsigned int crc_pcl(const u8 *buffer, int len, +asmlinkage unsigned int crc_pcl(const u8 *buffer, unsigned int len, unsigned int crc_init); #endif /* CONFIG_X86_64 */ diff --git a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S index bbcff1fb78cb..752812bc4991 100644 --- a/arch/x86/crypto/crc32c-pcl-intel-asm_64.S +++ b/arch/x86/crypto/crc32c-pcl-intel-asm_64.S @@ -7,6 +7,7 @@ * http://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-paper.pdf * * Copyright (C) 2012 Intel Corporation. + * Copyright 2024 Google LLC * * Authors: * Wajdi Feghali <wajdi.k.feghali@intel.com> @@ -44,185 +45,129 @@ */ #include <linux/linkage.h> -#include <asm/nospec-branch.h> ## ISCSI CRC 32 Implementation with crc32 and pclmulqdq Instruction -.macro LABEL prefix n -.L\prefix\n\(): -.endm - -.macro JMPTBL_ENTRY i -.quad .Lcrc_\i -.endm - -.macro JNC_LESS_THAN j - jnc .Lless_than_\j -.endm - -# Define threshold where buffers are considered "small" and routed to more -# efficient "by-1" code. This "by-1" code only handles up to 255 bytes, so -# SMALL_SIZE can be no larger than 255. - +# Define threshold below which buffers are considered "small" and routed to +# regular CRC code that does not interleave the CRC instructions. #define SMALL_SIZE 200 -.if (SMALL_SIZE > 255) -.error "SMALL_ SIZE must be < 256" -.endif - -# unsigned int crc_pcl(u8 *buffer, int len, unsigned int crc_init); +# unsigned int crc_pcl(const u8 *buffer, unsigned int len, unsigned int crc_init); .text SYM_FUNC_START(crc_pcl) -#define bufp rdi -#define bufp_dw %edi -#define bufp_w %di -#define bufp_b %dil -#define bufptmp %rcx -#define block_0 %rcx -#define block_1 %rdx -#define block_2 %r11 -#define len %rsi -#define len_dw %esi -#define len_w %si -#define len_b %sil -#define crc_init_arg %rdx -#define tmp %rbx -#define crc_init %r8 -#define crc_init_dw %r8d -#define crc1 %r9 -#define crc2 %r10 - - pushq %rbx - pushq %rdi - pushq %rsi - - ## Move crc_init for Linux to a different - mov crc_init_arg, crc_init +#define bufp %rdi +#define bufp_d %edi +#define len %esi +#define crc_init %edx +#define crc_init_q %rdx +#define n_misaligned %ecx /* overlaps chunk_bytes! */ +#define n_misaligned_q %rcx +#define chunk_bytes %ecx /* overlaps n_misaligned! */ +#define chunk_bytes_q %rcx +#define crc1 %r8 +#define crc2 %r9 + + cmp $SMALL_SIZE, len + jb .Lsmall ################################################################ ## 1) ALIGN: ################################################################ - - mov %bufp, bufptmp # rdi = *buf - neg %bufp - and $7, %bufp # calculate the unalignment amount of + mov bufp_d, n_misaligned + neg n_misaligned + and $7, n_misaligned # calculate the misalignment amount of # the address - je .Lproc_block # Skip if aligned - - ## If len is less than 8 and we're unaligned, we need to jump - ## to special code to avoid reading beyond the end of the buffer - cmp $8, len - jae .Ldo_align - # less_than_8 expects length in upper 3 bits of len_dw - # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30] - shl $32-3+1, len_dw - jmp .Lless_than_8_post_shl1 + je .Laligned # Skip if aligned + # Process 1 <= n_misaligned <= 7 bytes individually in order to align + # the remaining data to an 8-byte boundary. .Ldo_align: - #### Calculate CRC of unaligned bytes of the buffer (if any) - movq (bufptmp), tmp # load a quadward from the buffer - add %bufp, bufptmp # align buffer pointer for quadword - # processing - sub %bufp, len # update buffer length + movq (bufp), %rax + add n_misaligned_q, bufp + sub n_misaligned, len .Lalign_loop: - crc32b %bl, crc_init_dw # compute crc32 of 1-byte - shr $8, tmp # get next byte - dec %bufp + crc32b %al, crc_init # compute crc32 of 1-byte + shr $8, %rax # get next byte + dec n_misaligned jne .Lalign_loop - -.Lproc_block: +.Laligned: ################################################################ - ## 2) PROCESS BLOCKS: + ## 2) PROCESS BLOCK: ################################################################ - ## compute num of bytes to be processed - movq len, tmp # save num bytes in tmp - - cmpq $128*24, len + cmp $128*24, len jae .Lfull_block -.Lcontinue_block: - cmpq $SMALL_SIZE, len - jb .Lsmall - - ## len < 128*24 - movq $2731, %rax # 2731 = ceil(2^16 / 24) - mul len_dw - shrq $16, %rax - - ## eax contains floor(bytes / 24) = num 24-byte chunks to do - - ## process rax 24-byte chunks (128 >= rax >= 0) - - ## compute end address of each block - ## block 0 (base addr + RAX * 8) - ## block 1 (base addr + RAX * 16) - ## block 2 (base addr + RAX * 24) - lea (bufptmp, %rax, 8), block_0 - lea (block_0, %rax, 8), block_1 - lea (block_1, %rax, 8), block_2 +.Lpartial_block: + # Compute floor(len / 24) to get num qwords to process from each lane. + imul $2731, len, %eax # 2731 = ceil(2^16 / 24) + shr $16, %eax + jmp .Lcrc_3lanes - xor crc1, crc1 - xor crc2, crc2 - - ## branch into array - leaq jump_table(%rip), %bufp - mov (%bufp,%rax,8), %bufp - JMP_NOSPEC bufp - - ################################################################ - ## 2a) PROCESS FULL BLOCKS: - ################################################################ .Lfull_block: - movl $128,%eax - lea 128*8*2(block_0), block_1 - lea 128*8*3(block_0), block_2 - add $128*8*1, block_0 - - xor crc1,crc1 - xor crc2,crc2 - - # Fall through into top of crc array (crc_128) + # Processing 128 qwords from each lane. + mov $128, %eax ################################################################ - ## 3) CRC Array: + ## 3) CRC each of three lanes: ################################################################ - i=128 -.rept 128-1 -.altmacro -LABEL crc_ %i -.noaltmacro - ENDBR - crc32q -i*8(block_0), crc_init - crc32q -i*8(block_1), crc1 - crc32q -i*8(block_2), crc2 - i=(i-1) -.endr - -.altmacro -LABEL crc_ %i -.noaltmacro - ENDBR - crc32q -i*8(block_0), crc_init - crc32q -i*8(block_1), crc1 -# SKIP crc32 -i*8(block_2), crc2 ; Don't do this one yet - - mov block_2, block_0 +.Lcrc_3lanes: + xor crc1,crc1 + xor crc2,crc2 + mov %eax, chunk_bytes + shl $3, chunk_bytes # num bytes to process from each lane + sub $5, %eax # 4 for 4x_loop, 1 for special last iter + jl .Lcrc_3lanes_4x_done + + # Unroll the loop by a factor of 4 to reduce the overhead of the loop + # bookkeeping instructions, which can compete with crc32q for the ALUs. +.Lcrc_3lanes_4x_loop: + crc32q (bufp), crc_init_q + crc32q (bufp,chunk_bytes_q), crc1 + crc32q (bufp,chunk_bytes_q,2), crc2 + crc32q 8(bufp), crc_init_q + crc32q 8(bufp,chunk_bytes_q), crc1 + crc32q 8(bufp,chunk_bytes_q,2), crc2 + crc32q 16(bufp), crc_init_q + crc32q 16(bufp,chunk_bytes_q), crc1 + crc32q 16(bufp,chunk_bytes_q,2), crc2 + crc32q 24(bufp), crc_init_q + crc32q 24(bufp,chunk_bytes_q), crc1 + crc32q 24(bufp,chunk_bytes_q,2), crc2 + add $32, bufp + sub $4, %eax + jge .Lcrc_3lanes_4x_loop + +.Lcrc_3lanes_4x_done: + add $4, %eax + jz .Lcrc_3lanes_last_qword + +.Lcrc_3lanes_1x_loop: + crc32q (bufp), crc_init_q + crc32q (bufp,chunk_bytes_q), crc1 + crc32q (bufp,chunk_bytes_q,2), crc2 + add $8, bufp + dec %eax + jnz .Lcrc_3lanes_1x_loop + +.Lcrc_3lanes_last_qword: + crc32q (bufp), crc_init_q + crc32q (bufp,chunk_bytes_q), crc1 +# SKIP crc32q (bufp,chunk_bytes_q,2), crc2 ; Don't do this one yet ################################################################ ## 4) Combine three results: ################################################################ - lea (K_table-8)(%rip), %bufp # first entry is for idx 1 - shlq $3, %rax # rax *= 8 - pmovzxdq (%bufp,%rax), %xmm0 # 2 consts: K1:K2 - leal (%eax,%eax,2), %eax # rax *= 3 (total *24) - subq %rax, tmp # tmp -= rax*24 + lea (K_table-8)(%rip), %rax # first entry is for idx 1 + pmovzxdq (%rax,chunk_bytes_q), %xmm0 # 2 consts: K1:K2 + lea (chunk_bytes,chunk_bytes,2), %eax # chunk_bytes * 3 + sub %eax, len # len -= chunk_bytes * 3 - movq crc_init, %xmm1 # CRC for block 1 + movq crc_init_q, %xmm1 # CRC for block 1 pclmulqdq $0x00, %xmm0, %xmm1 # Multiply by K2 movq crc1, %xmm2 # CRC for block 2 @@ -230,103 +175,54 @@ LABEL crc_ %i pxor %xmm2,%xmm1 movq %xmm1, %rax - xor -i*8(block_2), %rax - mov crc2, crc_init - crc32 %rax, crc_init + xor (bufp,chunk_bytes_q,2), %rax + mov crc2, crc_init_q + crc32 %rax, crc_init_q + lea 8(bufp,chunk_bytes_q,2), bufp ################################################################ - ## 5) Check for end: + ## 5) If more blocks remain, goto (2): ################################################################ -LABEL crc_ 0 - ENDBR - mov tmp, len - cmp $128*24, tmp - jae .Lfull_block - cmp $24, tmp - jae .Lcontinue_block - -.Lless_than_24: - shl $32-4, len_dw # less_than_16 expects length - # in upper 4 bits of len_dw - jnc .Lless_than_16 - crc32q (bufptmp), crc_init - crc32q 8(bufptmp), crc_init - jz .Ldo_return - add $16, bufptmp - # len is less than 8 if we got here - # less_than_8 expects length in upper 3 bits of len_dw - # less_than_8_post_shl1 expects length = carryflag * 8 + len_dw[31:30] - shl $2, len_dw - jmp .Lless_than_8_post_shl1 + cmp $128*24, len + jae .Lfull_block + cmp $SMALL_SIZE, len + jae .Lpartial_block ####################################################################### - ## 6) LESS THAN 256-bytes REMAIN AT THIS POINT (8-bits of len are full) + ## 6) Process any remainder without interleaving: ####################################################################### .Lsmall: - shl $32-8, len_dw # Prepare len_dw for less_than_256 - j=256 -.rept 5 # j = {256, 128, 64, 32, 16} -.altmacro -LABEL less_than_ %j # less_than_j: Length should be in - # upper lg(j) bits of len_dw - j=(j/2) - shl $1, len_dw # Get next MSB - JNC_LESS_THAN %j -.noaltmacro - i=0 -.rept (j/8) - crc32q i(bufptmp), crc_init # Compute crc32 of 8-byte data - i=i+8 -.endr - jz .Ldo_return # Return if remaining length is zero - add $j, bufptmp # Advance buf -.endr - -.Lless_than_8: # Length should be stored in - # upper 3 bits of len_dw - shl $1, len_dw -.Lless_than_8_post_shl1: - jnc .Lless_than_4 - crc32l (bufptmp), crc_init_dw # CRC of 4 bytes - jz .Ldo_return # return if remaining data is zero - add $4, bufptmp -.Lless_than_4: # Length should be stored in - # upper 2 bits of len_dw - shl $1, len_dw - jnc .Lless_than_2 - crc32w (bufptmp), crc_init_dw # CRC of 2 bytes - jz .Ldo_return # return if remaining data is zero - add $2, bufptmp -.Lless_than_2: # Length should be stored in the MSB - # of len_dw - shl $1, len_dw - jnc .Lless_than_1 - crc32b (bufptmp), crc_init_dw # CRC of 1 byte -.Lless_than_1: # Length should be zero -.Ldo_return: - movq crc_init, %rax - popq %rsi - popq %rdi - popq %rbx + test len, len + jz .Ldone + mov len, %eax + shr $3, %eax + jz .Ldo_dword +.Ldo_qwords: + crc32q (bufp), crc_init_q + add $8, bufp + dec %eax + jnz .Ldo_qwords +.Ldo_dword: + test $4, len + jz .Ldo_word + crc32l (bufp), crc_init + add $4, bufp +.Ldo_word: + test $2, len + jz .Ldo_byte + crc32w (bufp), crc_init + add $2, bufp +.Ldo_byte: + test $1, len + jz .Ldone + crc32b (bufp), crc_init +.Ldone: + mov crc_init, %eax RET SYM_FUNC_END(crc_pcl) .section .rodata, "a", @progbits - ################################################################ - ## jump table Table is 129 entries x 2 bytes each - ################################################################ -.align 4 -jump_table: - i=0 -.rept 129 -.altmacro -JMPTBL_ENTRY %i -.noaltmacro - i=i+1 -.endr - - ################################################################ ## PCLMULQDQ tables ## Table is 128 entries x 2 words (8 bytes) each diff --git a/crypto/Kconfig b/crypto/Kconfig index a779cab668c2..6b0bfbccac08 100644 --- a/crypto/Kconfig +++ b/crypto/Kconfig @@ -250,6 +250,7 @@ config CRYPTO_RSA tristate "RSA (Rivest-Shamir-Adleman)" select CRYPTO_AKCIPHER select CRYPTO_MANAGER + select CRYPTO_SIG select MPILIB select ASN1 help @@ -290,19 +291,19 @@ config CRYPTO_ECDH config CRYPTO_ECDSA tristate "ECDSA (Elliptic Curve Digital Signature Algorithm)" select CRYPTO_ECC - select CRYPTO_AKCIPHER + select CRYPTO_SIG select ASN1 help ECDSA (Elliptic Curve Digital Signature Algorithm) (FIPS 186, ISO/IEC 14888-3) - using curves P-192, P-256, and P-384 + using curves P-192, P-256, P-384 and P-521 Only signature verification is implemented. config CRYPTO_ECRDSA tristate "EC-RDSA (Elliptic Curve Russian Digital Signature Algorithm)" select CRYPTO_ECC - select CRYPTO_AKCIPHER + select CRYPTO_SIG select CRYPTO_STREEBOG select OID_REGISTRY select ASN1 diff --git a/crypto/Makefile b/crypto/Makefile index 4c99e5d376f6..77abca715445 100644 --- a/crypto/Makefile +++ b/crypto/Makefile @@ -48,11 +48,14 @@ rsa_generic-y += rsaprivkey.asn1.o rsa_generic-y += rsa.o rsa_generic-y += rsa_helper.o rsa_generic-y += rsa-pkcs1pad.o +rsa_generic-y += rsassa-pkcs1.o obj-$(CONFIG_CRYPTO_RSA) += rsa_generic.o $(obj)/ecdsasignature.asn1.o: $(obj)/ecdsasignature.asn1.c $(obj)/ecdsasignature.asn1.h -$(obj)/ecdsa.o: $(obj)/ecdsasignature.asn1.h +$(obj)/ecdsa-x962.o: $(obj)/ecdsasignature.asn1.h ecdsa_generic-y += ecdsa.o +ecdsa_generic-y += ecdsa-x962.o +ecdsa_generic-y += ecdsa-p1363.o ecdsa_generic-y += ecdsasignature.asn1.o obj-$(CONFIG_CRYPTO_ECDSA) += ecdsa_generic.o @@ -152,6 +155,8 @@ obj-$(CONFIG_CRYPTO_DEFLATE) += deflate.o obj-$(CONFIG_CRYPTO_MICHAEL_MIC) += michael_mic.o obj-$(CONFIG_CRYPTO_CRC32C) += crc32c_generic.o obj-$(CONFIG_CRYPTO_CRC32) += crc32_generic.o +CFLAGS_crc32c_generic.o += -DARCH=$(ARCH) +CFLAGS_crc32_generic.o += -DARCH=$(ARCH) obj-$(CONFIG_CRYPTO_CRCT10DIF) += crct10dif_common.o crct10dif_generic.o obj-$(CONFIG_CRYPTO_CRC64_ROCKSOFT) += crc64_rocksoft_generic.o obj-$(CONFIG_CRYPTO_AUTHENC) += authenc.o authencesn.o diff --git a/crypto/akcipher.c b/crypto/akcipher.c index e0ff5f4dda6d..72c82d9aa077 100644 --- a/crypto/akcipher.c +++ b/crypto/akcipher.c @@ -20,6 +20,19 @@ #define CRYPTO_ALG_TYPE_AHASH_MASK 0x0000000e +struct crypto_akcipher_sync_data { + struct crypto_akcipher *tfm; + const void *src; + void *dst; + unsigned int slen; + unsigned int dlen; + + struct akcipher_request *req; + struct crypto_wait cwait; + struct scatterlist sg; + u8 *buf; +}; + static int __maybe_unused crypto_akcipher_report( struct sk_buff *skb, struct crypto_alg *alg) { @@ -126,10 +139,6 @@ int crypto_register_akcipher(struct akcipher_alg *alg) { struct crypto_alg *base = &alg->base; - if (!alg->sign) - alg->sign = akcipher_default_op; - if (!alg->verify) - alg->verify = akcipher_default_op; if (!alg->encrypt) alg->encrypt = akcipher_default_op; if (!alg->decrypt) @@ -158,7 +167,7 @@ int akcipher_register_instance(struct crypto_template *tmpl, } EXPORT_SYMBOL_GPL(akcipher_register_instance); -int crypto_akcipher_sync_prep(struct crypto_akcipher_sync_data *data) +static int crypto_akcipher_sync_prep(struct crypto_akcipher_sync_data *data) { unsigned int reqsize = crypto_akcipher_reqsize(data->tfm); struct akcipher_request *req; @@ -167,10 +176,7 @@ int crypto_akcipher_sync_prep(struct crypto_akcipher_sync_data *data) unsigned int len; u8 *buf; - if (data->dst) - mlen = max(data->slen, data->dlen); - else - mlen = data->slen + data->dlen; + mlen = max(data->slen, data->dlen); len = sizeof(*req) + reqsize + mlen; if (len < mlen) @@ -189,8 +195,7 @@ int crypto_akcipher_sync_prep(struct crypto_akcipher_sync_data *data) sg = &data->sg; sg_init_one(sg, buf, mlen); - akcipher_request_set_crypt(req, sg, data->dst ? sg : NULL, - data->slen, data->dlen); + akcipher_request_set_crypt(req, sg, sg, data->slen, data->dlen); crypto_init_wait(&data->cwait); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, @@ -198,18 +203,16 @@ int crypto_akcipher_sync_prep(struct crypto_akcipher_sync_data *data) return 0; } -EXPORT_SYMBOL_GPL(crypto_akcipher_sync_prep); -int crypto_akcipher_sync_post(struct crypto_akcipher_sync_data *data, int err) +static int crypto_akcipher_sync_post(struct crypto_akcipher_sync_data *data, + int err) { err = crypto_wait_req(err, &data->cwait); - if (data->dst) - memcpy(data->dst, data->buf, data->dlen); + memcpy(data->dst, data->buf, data->dlen); data->dlen = data->req->dst_len; kfree_sensitive(data->req); return err; } -EXPORT_SYMBOL_GPL(crypto_akcipher_sync_post); int crypto_akcipher_sync_encrypt(struct crypto_akcipher *tfm, const void *src, unsigned int slen, @@ -248,34 +251,5 @@ int crypto_akcipher_sync_decrypt(struct crypto_akcipher *tfm, } EXPORT_SYMBOL_GPL(crypto_akcipher_sync_decrypt); -static void crypto_exit_akcipher_ops_sig(struct crypto_tfm *tfm) -{ - struct crypto_akcipher **ctx = crypto_tfm_ctx(tfm); - - crypto_free_akcipher(*ctx); -} - -int crypto_init_akcipher_ops_sig(struct crypto_tfm *tfm) -{ - struct crypto_akcipher **ctx = crypto_tfm_ctx(tfm); - struct crypto_alg *calg = tfm->__crt_alg; - struct crypto_akcipher *akcipher; - - if (!crypto_mod_get(calg)) - return -EAGAIN; - - akcipher = crypto_create_tfm(calg, &crypto_akcipher_type); - if (IS_ERR(akcipher)) { - crypto_mod_put(calg); - return PTR_ERR(akcipher); - } - - *ctx = akcipher; - tfm->exit = crypto_exit_akcipher_ops_sig; - - return 0; -} -EXPORT_SYMBOL_GPL(crypto_init_akcipher_ops_sig); - MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Generic public key cipher type"); diff --git a/crypto/algapi.c b/crypto/algapi.c index 004d27e41315..16f7c7a9d8ab 100644 --- a/crypto/algapi.c +++ b/crypto/algapi.c @@ -6,7 +6,6 @@ */ #include <crypto/algapi.h> -#include <crypto/internal/simd.h> #include <linux/err.h> #include <linux/errno.h> #include <linux/fips.h> @@ -23,11 +22,6 @@ static LIST_HEAD(crypto_template_list); -#ifdef CONFIG_CRYPTO_MANAGER_EXTRA_TESTS -DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test); -EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test); -#endif - static inline void crypto_check_module_sig(struct module *mod) { if (fips_enabled && mod && !module_sig_ok(mod)) diff --git a/crypto/asymmetric_keys/public_key.c b/crypto/asymmetric_keys/public_key.c index 422940a6706a..bbd07a9022e6 100644 --- a/crypto/asymmetric_keys/public_key.c +++ b/crypto/asymmetric_keys/public_key.c @@ -83,13 +83,19 @@ software_key_determine_akcipher(const struct public_key *pkey, if (strcmp(encoding, "pkcs1") == 0) { *sig = op == kernel_pkey_sign || op == kernel_pkey_verify; - if (!hash_algo) { + if (!*sig) { + /* + * For encrypt/decrypt, hash_algo is not used + * but allowed to be set for historic reasons. + */ n = snprintf(alg_name, CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)", pkey->pkey_algo); } else { + if (!hash_algo) + hash_algo = "none"; n = snprintf(alg_name, CRYPTO_MAX_ALG_NAME, - "pkcs1pad(%s,%s)", + "pkcs1(%s,%s)", pkey->pkey_algo, hash_algo); } return n >= CRYPTO_MAX_ALG_NAME ? -EINVAL : 0; @@ -104,7 +110,8 @@ software_key_determine_akcipher(const struct public_key *pkey, return -EINVAL; *sig = false; } else if (strncmp(pkey->pkey_algo, "ecdsa", 5) == 0) { - if (strcmp(encoding, "x962") != 0) + if (strcmp(encoding, "x962") != 0 && + strcmp(encoding, "p1363") != 0) return -EINVAL; /* * ECDSA signatures are taken over a raw hash, so they don't @@ -124,6 +131,9 @@ software_key_determine_akcipher(const struct public_key *pkey, strcmp(hash_algo, "sha3-384") != 0 && strcmp(hash_algo, "sha3-512") != 0) return -EINVAL; + n = snprintf(alg_name, CRYPTO_MAX_ALG_NAME, "%s(%s)", + encoding, pkey->pkey_algo); + return n >= CRYPTO_MAX_ALG_NAME ? -EINVAL : 0; } else if (strcmp(pkey->pkey_algo, "ecrdsa") == 0) { if (strcmp(encoding, "raw") != 0) return -EINVAL; @@ -192,7 +202,9 @@ static int software_key_query(const struct kernel_pkey_params *params, if (ret < 0) goto error_free_tfm; - len = crypto_sig_maxsize(sig); + len = crypto_sig_keysize(sig); + info->max_sig_size = crypto_sig_maxsize(sig); + info->max_data_size = crypto_sig_digestsize(sig); info->supported_ops = KEYCTL_SUPPORTS_VERIFY; if (pkey->key_is_private) @@ -218,6 +230,8 @@ static int software_key_query(const struct kernel_pkey_params *params, goto error_free_tfm; len = crypto_akcipher_maxsize(tfm); + info->max_sig_size = len; + info->max_data_size = len; info->supported_ops = KEYCTL_SUPPORTS_ENCRYPT; if (pkey->key_is_private) @@ -225,40 +239,6 @@ static int software_key_query(const struct kernel_pkey_params *params, } info->key_size = len * 8; - - if (strncmp(pkey->pkey_algo, "ecdsa", 5) == 0) { - int slen = len; - /* - * ECDSA key sizes are much smaller than RSA, and thus could - * operate on (hashed) inputs that are larger than key size. - * For example SHA384-hashed input used with secp256r1 - * based keys. Set max_data_size to be at least as large as - * the largest supported hash size (SHA512) - */ - info->max_data_size = 64; - - /* - * Verify takes ECDSA-Sig (described in RFC 5480) as input, - * which is actually 2 'key_size'-bit integers encoded in - * ASN.1. Account for the ASN.1 encoding overhead here. - * - * NIST P192/256/384 may prepend a '0' to a coordinate to - * indicate a positive integer. NIST P521 never needs it. - */ - if (strcmp(pkey->pkey_algo, "ecdsa-nist-p521") != 0) - slen += 1; - /* Length of encoding the x & y coordinates */ - slen = 2 * (slen + 2); - /* - * If coordinate encoding takes at least 128 bytes then an - * additional byte for length encoding is needed. - */ - info->max_sig_size = 1 + (slen >= 128) + 1 + slen; - } else { - info->max_data_size = len; - info->max_sig_size = len; - } - info->max_enc_size = len; info->max_dec_size = len; @@ -323,7 +303,7 @@ static int software_key_eds_op(struct kernel_pkey_params *params, if (ret) goto error_free_tfm; - ksz = crypto_sig_maxsize(sig); + ksz = crypto_sig_keysize(sig); } else { tfm = crypto_alloc_akcipher(alg_name, 0, 0); if (IS_ERR(tfm)) { diff --git a/crypto/asymmetric_keys/signature.c b/crypto/asymmetric_keys/signature.c index 2deff81f8af5..041d04b5c953 100644 --- a/crypto/asymmetric_keys/signature.c +++ b/crypto/asymmetric_keys/signature.c @@ -65,69 +65,6 @@ int query_asymmetric_key(const struct kernel_pkey_params *params, EXPORT_SYMBOL_GPL(query_asymmetric_key); /** - * encrypt_blob - Encrypt data using an asymmetric key - * @params: Various parameters - * @data: Data blob to be encrypted, length params->data_len - * @enc: Encrypted data buffer, length params->enc_len - * - * Encrypt the specified data blob using the private key specified by - * params->key. The encrypted data is wrapped in an encoding if - * params->encoding is specified (eg. "pkcs1"). - * - * Returns the length of the data placed in the encrypted data buffer or an - * error. - */ -int encrypt_blob(struct kernel_pkey_params *params, - const void *data, void *enc) -{ - params->op = kernel_pkey_encrypt; - return asymmetric_key_eds_op(params, data, enc); -} -EXPORT_SYMBOL_GPL(encrypt_blob); - -/** - * decrypt_blob - Decrypt data using an asymmetric key - * @params: Various parameters - * @enc: Encrypted data to be decrypted, length params->enc_len - * @data: Decrypted data buffer, length params->data_len - * - * Decrypt the specified data blob using the private key specified by - * params->key. The decrypted data is wrapped in an encoding if - * params->encoding is specified (eg. "pkcs1"). - * - * Returns the length of the data placed in the decrypted data buffer or an - * error. - */ -int decrypt_blob(struct kernel_pkey_params *params, - const void *enc, void *data) -{ - params->op = kernel_pkey_decrypt; - return asymmetric_key_eds_op(params, enc, data); -} -EXPORT_SYMBOL_GPL(decrypt_blob); - -/** - * create_signature - Sign some data using an asymmetric key - * @params: Various parameters - * @data: Data blob to be signed, length params->data_len - * @enc: Signature buffer, length params->enc_len - * - * Sign the specified data blob using the private key specified by params->key. - * The signature is wrapped in an encoding if params->encoding is specified - * (eg. "pkcs1"). If the encoding needs to know the digest type, this can be - * passed through params->hash_algo (eg. "sha1"). - * - * Returns the length of the data placed in the signature buffer or an error. - */ -int create_signature(struct kernel_pkey_params *params, - const void *data, void *enc) -{ - params->op = kernel_pkey_sign; - return asymmetric_key_eds_op(params, data, enc); -} -EXPORT_SYMBOL_GPL(create_signature); - -/** * verify_signature - Initiate the use of an asymmetric key to verify a signature * @key: The asymmetric key to verify against * @sig: The signature to check diff --git a/crypto/crc32_generic.c b/crypto/crc32_generic.c index d1251663ed66..6a55d206fab3 100644 --- a/crypto/crc32_generic.c +++ b/crypto/crc32_generic.c @@ -59,6 +59,15 @@ static int crc32_update(struct shash_desc *desc, const u8 *data, { u32 *crcp = shash_desc_ctx(desc); + *crcp = crc32_le_base(*crcp, data, len); + return 0; +} + +static int crc32_update_arch(struct shash_desc *desc, const u8 *data, + unsigned int len) +{ + u32 *crcp = shash_desc_ctx(desc); + *crcp = crc32_le(*crcp, data, len); return 0; } @@ -67,6 +76,13 @@ static int crc32_update(struct shash_desc *desc, const u8 *data, static int __crc32_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out) { + put_unaligned_le32(crc32_le_base(*crcp, data, len), out); + return 0; +} + +static int __crc32_finup_arch(u32 *crcp, const u8 *data, unsigned int len, + u8 *out) +{ put_unaligned_le32(crc32_le(*crcp, data, len), out); return 0; } @@ -77,6 +93,12 @@ static int crc32_finup(struct shash_desc *desc, const u8 *data, return __crc32_finup(shash_desc_ctx(desc), data, len, out); } +static int crc32_finup_arch(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + return __crc32_finup_arch(shash_desc_ctx(desc), data, len, out); +} + static int crc32_final(struct shash_desc *desc, u8 *out) { u32 *crcp = shash_desc_ctx(desc); @@ -88,38 +110,62 @@ static int crc32_final(struct shash_desc *desc, u8 *out) static int crc32_digest(struct shash_desc *desc, const u8 *data, unsigned int len, u8 *out) { - return __crc32_finup(crypto_shash_ctx(desc->tfm), data, len, - out); + return __crc32_finup(crypto_shash_ctx(desc->tfm), data, len, out); } -static struct shash_alg alg = { - .setkey = crc32_setkey, - .init = crc32_init, - .update = crc32_update, - .final = crc32_final, - .finup = crc32_finup, - .digest = crc32_digest, - .descsize = sizeof(u32), - .digestsize = CHKSUM_DIGEST_SIZE, - .base = { - .cra_name = "crc32", - .cra_driver_name = "crc32-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .cra_blocksize = CHKSUM_BLOCK_SIZE, - .cra_ctxsize = sizeof(u32), - .cra_module = THIS_MODULE, - .cra_init = crc32_cra_init, - } -}; + +static int crc32_digest_arch(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + return __crc32_finup_arch(crypto_shash_ctx(desc->tfm), data, len, out); +} + +static struct shash_alg algs[] = {{ + .setkey = crc32_setkey, + .init = crc32_init, + .update = crc32_update, + .final = crc32_final, + .finup = crc32_finup, + .digest = crc32_digest, + .descsize = sizeof(u32), + .digestsize = CHKSUM_DIGEST_SIZE, + + .base.cra_name = "crc32", + .base.cra_driver_name = "crc32-generic", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .base.cra_blocksize = CHKSUM_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(u32), + .base.cra_module = THIS_MODULE, + .base.cra_init = crc32_cra_init, +}, { + .setkey = crc32_setkey, + .init = crc32_init, + .update = crc32_update_arch, + .final = crc32_final, + .finup = crc32_finup_arch, + .digest = crc32_digest_arch, + .descsize = sizeof(u32), + .digestsize = CHKSUM_DIGEST_SIZE, + + .base.cra_name = "crc32", + .base.cra_driver_name = "crc32-" __stringify(ARCH), + .base.cra_priority = 150, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .base.cra_blocksize = CHKSUM_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(u32), + .base.cra_module = THIS_MODULE, + .base.cra_init = crc32_cra_init, +}}; static int __init crc32_mod_init(void) { - return crypto_register_shash(&alg); + /* register the arch flavor only if it differs from the generic one */ + return crypto_register_shashes(algs, 1 + (&crc32_le != &crc32_le_base)); } static void __exit crc32_mod_fini(void) { - crypto_unregister_shash(&alg); + crypto_unregister_shashes(algs, 1 + (&crc32_le != &crc32_le_base)); } subsys_initcall(crc32_mod_init); diff --git a/crypto/crc32c_generic.c b/crypto/crc32c_generic.c index a8c90b3f4c6c..7c2357c30fdf 100644 --- a/crypto/crc32c_generic.c +++ b/crypto/crc32c_generic.c @@ -85,6 +85,15 @@ static int chksum_update(struct shash_desc *desc, const u8 *data, { struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + ctx->crc = __crc32c_le_base(ctx->crc, data, length); + return 0; +} + +static int chksum_update_arch(struct shash_desc *desc, const u8 *data, + unsigned int length) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + ctx->crc = __crc32c_le(ctx->crc, data, length); return 0; } @@ -99,6 +108,13 @@ static int chksum_final(struct shash_desc *desc, u8 *out) static int __chksum_finup(u32 *crcp, const u8 *data, unsigned int len, u8 *out) { + put_unaligned_le32(~__crc32c_le_base(*crcp, data, len), out); + return 0; +} + +static int __chksum_finup_arch(u32 *crcp, const u8 *data, unsigned int len, + u8 *out) +{ put_unaligned_le32(~__crc32c_le(*crcp, data, len), out); return 0; } @@ -111,6 +127,14 @@ static int chksum_finup(struct shash_desc *desc, const u8 *data, return __chksum_finup(&ctx->crc, data, len, out); } +static int chksum_finup_arch(struct shash_desc *desc, const u8 *data, + unsigned int len, u8 *out) +{ + struct chksum_desc_ctx *ctx = shash_desc_ctx(desc); + + return __chksum_finup_arch(&ctx->crc, data, len, out); +} + static int chksum_digest(struct shash_desc *desc, const u8 *data, unsigned int length, u8 *out) { @@ -119,6 +143,14 @@ static int chksum_digest(struct shash_desc *desc, const u8 *data, return __chksum_finup(&mctx->key, data, length, out); } +static int chksum_digest_arch(struct shash_desc *desc, const u8 *data, + unsigned int length, u8 *out) +{ + struct chksum_ctx *mctx = crypto_shash_ctx(desc->tfm); + + return __chksum_finup_arch(&mctx->key, data, length, out); +} + static int crc32c_cra_init(struct crypto_tfm *tfm) { struct chksum_ctx *mctx = crypto_tfm_ctx(tfm); @@ -127,35 +159,53 @@ static int crc32c_cra_init(struct crypto_tfm *tfm) return 0; } -static struct shash_alg alg = { - .digestsize = CHKSUM_DIGEST_SIZE, - .setkey = chksum_setkey, - .init = chksum_init, - .update = chksum_update, - .final = chksum_final, - .finup = chksum_finup, - .digest = chksum_digest, - .descsize = sizeof(struct chksum_desc_ctx), - .base = { - .cra_name = "crc32c", - .cra_driver_name = "crc32c-generic", - .cra_priority = 100, - .cra_flags = CRYPTO_ALG_OPTIONAL_KEY, - .cra_blocksize = CHKSUM_BLOCK_SIZE, - .cra_ctxsize = sizeof(struct chksum_ctx), - .cra_module = THIS_MODULE, - .cra_init = crc32c_cra_init, - } -}; +static struct shash_alg algs[] = {{ + .digestsize = CHKSUM_DIGEST_SIZE, + .setkey = chksum_setkey, + .init = chksum_init, + .update = chksum_update, + .final = chksum_final, + .finup = chksum_finup, + .digest = chksum_digest, + .descsize = sizeof(struct chksum_desc_ctx), + + .base.cra_name = "crc32c", + .base.cra_driver_name = "crc32c-generic", + .base.cra_priority = 100, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .base.cra_blocksize = CHKSUM_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct chksum_ctx), + .base.cra_module = THIS_MODULE, + .base.cra_init = crc32c_cra_init, +}, { + .digestsize = CHKSUM_DIGEST_SIZE, + .setkey = chksum_setkey, + .init = chksum_init, + .update = chksum_update_arch, + .final = chksum_final, + .finup = chksum_finup_arch, + .digest = chksum_digest_arch, + .descsize = sizeof(struct chksum_desc_ctx), + + .base.cra_name = "crc32c", + .base.cra_driver_name = "crc32c-" __stringify(ARCH), + .base.cra_priority = 150, + .base.cra_flags = CRYPTO_ALG_OPTIONAL_KEY, + .base.cra_blocksize = CHKSUM_BLOCK_SIZE, + .base.cra_ctxsize = sizeof(struct chksum_ctx), + .base.cra_module = THIS_MODULE, + .base.cra_init = crc32c_cra_init, +}}; static int __init crc32c_mod_init(void) { - return crypto_register_shash(&alg); + /* register the arch flavor only if it differs from the generic one */ + return crypto_register_shashes(algs, 1 + (&__crc32c_le != &__crc32c_le_base)); } static void __exit crc32c_mod_fini(void) { - crypto_unregister_shash(&alg); + crypto_unregister_shashes(algs, 1 + (&__crc32c_le != &__crc32c_le_base)); } subsys_initcall(crc32c_mod_init); diff --git a/crypto/drbg.c b/crypto/drbg.c index 3addce90930c..c323f40bed4f 100644 --- a/crypto/drbg.c +++ b/crypto/drbg.c @@ -101,6 +101,7 @@ #include <crypto/internal/cipher.h> #include <linux/kernel.h> #include <linux/jiffies.h> +#include <linux/string_choices.h> /*************************************************************** * Backend cipher definitions available to DRBG @@ -1412,7 +1413,7 @@ static int drbg_generate(struct drbg_state *drbg, if (drbg->pr || drbg->seeded == DRBG_SEED_STATE_UNSEEDED) { pr_devel("DRBG: reseeding before generation (prediction " "resistance: %s, state %s)\n", - drbg->pr ? "true" : "false", + str_true_false(drbg->pr), (drbg->seeded == DRBG_SEED_STATE_FULL ? "seeded" : "unseeded")); /* 9.3.1 steps 7.1 through 7.3 */ @@ -1562,7 +1563,7 @@ static int drbg_instantiate(struct drbg_state *drbg, struct drbg_string *pers, bool reseed = true; pr_devel("DRBG: Initializing DRBG core %d with prediction resistance " - "%s\n", coreref, pr ? "enabled" : "disabled"); + "%s\n", coreref, str_enabled_disabled(pr)); mutex_lock(&drbg->drbg_mutex); /* 9.1 step 1 is implicit with the selected DRBG type */ diff --git a/crypto/ecdsa-p1363.c b/crypto/ecdsa-p1363.c new file mode 100644 index 000000000000..eaae7214d69b --- /dev/null +++ b/crypto/ecdsa-p1363.c @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * ECDSA P1363 signature encoding + * + * Copyright (c) 2024 Intel Corporation + */ + +#include <linux/err.h> +#include <linux/module.h> +#include <crypto/algapi.h> +#include <crypto/sig.h> +#include <crypto/internal/ecc.h> +#include <crypto/internal/sig.h> + +struct ecdsa_p1363_ctx { + struct crypto_sig *child; +}; + +static int ecdsa_p1363_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); + unsigned int keylen = crypto_sig_keysize(ctx->child); + unsigned int ndigits = DIV_ROUND_UP(keylen, sizeof(u64)); + struct ecdsa_raw_sig sig; + + if (slen != 2 * keylen) + return -EINVAL; + + ecc_digits_from_bytes(src, keylen, sig.r, ndigits); + ecc_digits_from_bytes(src + keylen, keylen, sig.s, ndigits); + + return crypto_sig_verify(ctx->child, &sig, sizeof(sig), digest, dlen); +} + +static unsigned int ecdsa_p1363_key_size(struct crypto_sig *tfm) +{ + struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); + + return crypto_sig_keysize(ctx->child); +} + +static unsigned int ecdsa_p1363_max_size(struct crypto_sig *tfm) +{ + struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); + + return 2 * crypto_sig_keysize(ctx->child); +} + +static unsigned int ecdsa_p1363_digest_size(struct crypto_sig *tfm) +{ + struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); + + return crypto_sig_digestsize(ctx->child); +} + +static int ecdsa_p1363_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); + + return crypto_sig_set_pubkey(ctx->child, key, keylen); +} + +static int ecdsa_p1363_init_tfm(struct crypto_sig *tfm) +{ + struct sig_instance *inst = sig_alg_instance(tfm); + struct crypto_sig_spawn *spawn = sig_instance_ctx(inst); + struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); + struct crypto_sig *child_tfm; + + child_tfm = crypto_spawn_sig(spawn); + if (IS_ERR(child_tfm)) + return PTR_ERR(child_tfm); + + ctx->child = child_tfm; + + return 0; +} + +static void ecdsa_p1363_exit_tfm(struct crypto_sig *tfm) +{ + struct ecdsa_p1363_ctx *ctx = crypto_sig_ctx(tfm); + + crypto_free_sig(ctx->child); +} + +static void ecdsa_p1363_free(struct sig_instance *inst) +{ + struct crypto_sig_spawn *spawn = sig_instance_ctx(inst); + + crypto_drop_sig(spawn); + kfree(inst); +} + +static int ecdsa_p1363_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct crypto_sig_spawn *spawn; + struct sig_instance *inst; + struct sig_alg *ecdsa_alg; + u32 mask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SIG, &mask); + if (err) + return err; + + inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + if (!inst) + return -ENOMEM; + + spawn = sig_instance_ctx(inst); + + err = crypto_grab_sig(spawn, sig_crypto_instance(inst), + crypto_attr_alg_name(tb[1]), 0, mask); + if (err) + goto err_free_inst; + + ecdsa_alg = crypto_spawn_sig_alg(spawn); + + err = -EINVAL; + if (strncmp(ecdsa_alg->base.cra_name, "ecdsa", 5) != 0) + goto err_free_inst; + + err = crypto_inst_setname(sig_crypto_instance(inst), tmpl->name, + &ecdsa_alg->base); + if (err) + goto err_free_inst; + + inst->alg.base.cra_priority = ecdsa_alg->base.cra_priority; + inst->alg.base.cra_ctxsize = sizeof(struct ecdsa_p1363_ctx); + + inst->alg.init = ecdsa_p1363_init_tfm; + inst->alg.exit = ecdsa_p1363_exit_tfm; + + inst->alg.verify = ecdsa_p1363_verify; + inst->alg.key_size = ecdsa_p1363_key_size; + inst->alg.max_size = ecdsa_p1363_max_size; + inst->alg.digest_size = ecdsa_p1363_digest_size; + inst->alg.set_pub_key = ecdsa_p1363_set_pub_key; + + inst->free = ecdsa_p1363_free; + + err = sig_register_instance(tmpl, inst); + if (err) { +err_free_inst: + ecdsa_p1363_free(inst); + } + return err; +} + +struct crypto_template ecdsa_p1363_tmpl = { + .name = "p1363", + .create = ecdsa_p1363_create, + .module = THIS_MODULE, +}; + +MODULE_ALIAS_CRYPTO("p1363"); diff --git a/crypto/ecdsa-x962.c b/crypto/ecdsa-x962.c new file mode 100644 index 000000000000..6a77c13e192b --- /dev/null +++ b/crypto/ecdsa-x962.c @@ -0,0 +1,237 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * ECDSA X9.62 signature encoding + * + * Copyright (c) 2021 IBM Corporation + * Copyright (c) 2024 Intel Corporation + */ + +#include <linux/asn1_decoder.h> +#include <linux/err.h> +#include <linux/module.h> +#include <crypto/algapi.h> +#include <crypto/sig.h> +#include <crypto/internal/ecc.h> +#include <crypto/internal/sig.h> + +#include "ecdsasignature.asn1.h" + +struct ecdsa_x962_ctx { + struct crypto_sig *child; +}; + +struct ecdsa_x962_signature_ctx { + struct ecdsa_raw_sig sig; + unsigned int ndigits; +}; + +/* Get the r and s components of a signature from the X.509 certificate. */ +static int ecdsa_get_signature_rs(u64 *dest, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen, + unsigned int ndigits) +{ + size_t bufsize = ndigits * sizeof(u64); + const char *d = value; + + if (!value || !vlen || vlen > bufsize + 1) + return -EINVAL; + + /* + * vlen may be 1 byte larger than bufsize due to a leading zero byte + * (necessary if the most significant bit of the integer is set). + */ + if (vlen > bufsize) { + /* skip over leading zeros that make 'value' a positive int */ + if (*d == 0) { + vlen -= 1; + d++; + } else { + return -EINVAL; + } + } + + ecc_digits_from_bytes(d, vlen, dest, ndigits); + + return 0; +} + +int ecdsa_get_signature_r(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecdsa_x962_signature_ctx *sig_ctx = context; + + return ecdsa_get_signature_rs(sig_ctx->sig.r, hdrlen, tag, value, vlen, + sig_ctx->ndigits); +} + +int ecdsa_get_signature_s(void *context, size_t hdrlen, unsigned char tag, + const void *value, size_t vlen) +{ + struct ecdsa_x962_signature_ctx *sig_ctx = context; + + return ecdsa_get_signature_rs(sig_ctx->sig.s, hdrlen, tag, value, vlen, + sig_ctx->ndigits); +} + +static int ecdsa_x962_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); + struct ecdsa_x962_signature_ctx sig_ctx; + int err; + + sig_ctx.ndigits = DIV_ROUND_UP(crypto_sig_keysize(ctx->child), + sizeof(u64)); + + err = asn1_ber_decoder(&ecdsasignature_decoder, &sig_ctx, src, slen); + if (err < 0) + return err; + + return crypto_sig_verify(ctx->child, &sig_ctx.sig, sizeof(sig_ctx.sig), + digest, dlen); +} + +static unsigned int ecdsa_x962_key_size(struct crypto_sig *tfm) +{ + struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); + + return crypto_sig_keysize(ctx->child); +} + +static unsigned int ecdsa_x962_max_size(struct crypto_sig *tfm) +{ + struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); + struct sig_alg *alg = crypto_sig_alg(ctx->child); + int slen = crypto_sig_keysize(ctx->child); + + /* + * Verify takes ECDSA-Sig-Value (described in RFC 5480) as input, + * which is actually 2 'key_size'-bit integers encoded in ASN.1. + * Account for the ASN.1 encoding overhead here. + * + * NIST P192/256/384 may prepend a '0' to a coordinate to indicate + * a positive integer. NIST P521 never needs it. + */ + if (strcmp(alg->base.cra_name, "ecdsa-nist-p521") != 0) + slen += 1; + + /* Length of encoding the x & y coordinates */ + slen = 2 * (slen + 2); + + /* + * If coordinate encoding takes at least 128 bytes then an + * additional byte for length encoding is needed. + */ + return 1 + (slen >= 128) + 1 + slen; +} + +static unsigned int ecdsa_x962_digest_size(struct crypto_sig *tfm) +{ + struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); + + return crypto_sig_digestsize(ctx->child); +} + +static int ecdsa_x962_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); + + return crypto_sig_set_pubkey(ctx->child, key, keylen); +} + +static int ecdsa_x962_init_tfm(struct crypto_sig *tfm) +{ + struct sig_instance *inst = sig_alg_instance(tfm); + struct crypto_sig_spawn *spawn = sig_instance_ctx(inst); + struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); + struct crypto_sig *child_tfm; + + child_tfm = crypto_spawn_sig(spawn); + if (IS_ERR(child_tfm)) + return PTR_ERR(child_tfm); + + ctx->child = child_tfm; + + return 0; +} + +static void ecdsa_x962_exit_tfm(struct crypto_sig *tfm) +{ + struct ecdsa_x962_ctx *ctx = crypto_sig_ctx(tfm); + + crypto_free_sig(ctx->child); +} + +static void ecdsa_x962_free(struct sig_instance *inst) +{ + struct crypto_sig_spawn *spawn = sig_instance_ctx(inst); + + crypto_drop_sig(spawn); + kfree(inst); +} + +static int ecdsa_x962_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct crypto_sig_spawn *spawn; + struct sig_instance *inst; + struct sig_alg *ecdsa_alg; + u32 mask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SIG, &mask); + if (err) + return err; + + inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL); + if (!inst) + return -ENOMEM; + + spawn = sig_instance_ctx(inst); + + err = crypto_grab_sig(spawn, sig_crypto_instance(inst), + crypto_attr_alg_name(tb[1]), 0, mask); + if (err) + goto err_free_inst; + + ecdsa_alg = crypto_spawn_sig_alg(spawn); + + err = -EINVAL; + if (strncmp(ecdsa_alg->base.cra_name, "ecdsa", 5) != 0) + goto err_free_inst; + + err = crypto_inst_setname(sig_crypto_instance(inst), tmpl->name, + &ecdsa_alg->base); + if (err) + goto err_free_inst; + + inst->alg.base.cra_priority = ecdsa_alg->base.cra_priority; + inst->alg.base.cra_ctxsize = sizeof(struct ecdsa_x962_ctx); + + inst->alg.init = ecdsa_x962_init_tfm; + inst->alg.exit = ecdsa_x962_exit_tfm; + + inst->alg.verify = ecdsa_x962_verify; + inst->alg.key_size = ecdsa_x962_key_size; + inst->alg.max_size = ecdsa_x962_max_size; + inst->alg.digest_size = ecdsa_x962_digest_size; + inst->alg.set_pub_key = ecdsa_x962_set_pub_key; + + inst->free = ecdsa_x962_free; + + err = sig_register_instance(tmpl, inst); + if (err) { +err_free_inst: + ecdsa_x962_free(inst); + } + return err; +} + +struct crypto_template ecdsa_x962_tmpl = { + .name = "x962", + .create = ecdsa_x962_create, + .module = THIS_MODULE, +}; + +MODULE_ALIAS_CRYPTO("x962"); diff --git a/crypto/ecdsa.c b/crypto/ecdsa.c index d5a10959ec28..117526d15dde 100644 --- a/crypto/ecdsa.c +++ b/crypto/ecdsa.c @@ -4,14 +4,11 @@ */ #include <linux/module.h> -#include <crypto/internal/akcipher.h> #include <crypto/internal/ecc.h> -#include <crypto/akcipher.h> +#include <crypto/internal/sig.h> #include <crypto/ecdh.h> -#include <linux/asn1_decoder.h> -#include <linux/scatterlist.h> - -#include "ecdsasignature.asn1.h" +#include <crypto/sha2.h> +#include <crypto/sig.h> struct ecc_ctx { unsigned int curve_id; @@ -23,66 +20,6 @@ struct ecc_ctx { struct ecc_point pub_key; }; -struct ecdsa_signature_ctx { - const struct ecc_curve *curve; - u64 r[ECC_MAX_DIGITS]; - u64 s[ECC_MAX_DIGITS]; -}; - -/* - * Get the r and s components of a signature from the X509 certificate. - */ -static int ecdsa_get_signature_rs(u64 *dest, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen, unsigned int ndigits) -{ - size_t bufsize = ndigits * sizeof(u64); - ssize_t diff = vlen - bufsize; - const char *d = value; - - if (!value || !vlen) - return -EINVAL; - - /* diff = 0: 'value' has exacly the right size - * diff > 0: 'value' has too many bytes; one leading zero is allowed that - * makes the value a positive integer; error on more - * diff < 0: 'value' is missing leading zeros - */ - if (diff > 0) { - /* skip over leading zeros that make 'value' a positive int */ - if (*d == 0) { - vlen -= 1; - diff--; - d++; - } - if (diff) - return -EINVAL; - } - if (-diff >= bufsize) - return -EINVAL; - - ecc_digits_from_bytes(d, vlen, dest, ndigits); - - return 0; -} - -int ecdsa_get_signature_r(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) -{ - struct ecdsa_signature_ctx *sig = context; - - return ecdsa_get_signature_rs(sig->r, hdrlen, tag, value, vlen, - sig->curve->g.ndigits); -} - -int ecdsa_get_signature_s(void *context, size_t hdrlen, unsigned char tag, - const void *value, size_t vlen) -{ - struct ecdsa_signature_ctx *sig = context; - - return ecdsa_get_signature_rs(sig->s, hdrlen, tag, value, vlen, - sig->curve->g.ndigits); -} - static int _ecdsa_verify(struct ecc_ctx *ctx, const u64 *hash, const u64 *r, const u64 *s) { const struct ecc_curve *curve = ctx->curve; @@ -126,46 +63,27 @@ static int _ecdsa_verify(struct ecc_ctx *ctx, const u64 *hash, const u64 *r, con /* * Verify an ECDSA signature. */ -static int ecdsa_verify(struct akcipher_request *req) +static int ecdsa_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) { - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); size_t bufsize = ctx->curve->g.ndigits * sizeof(u64); - struct ecdsa_signature_ctx sig_ctx = { - .curve = ctx->curve, - }; + const struct ecdsa_raw_sig *sig = src; u64 hash[ECC_MAX_DIGITS]; - unsigned char *buffer; - int ret; if (unlikely(!ctx->pub_key_set)) return -EINVAL; - buffer = kmalloc(req->src_len + req->dst_len, GFP_KERNEL); - if (!buffer) - return -ENOMEM; - - sg_pcopy_to_buffer(req->src, - sg_nents_for_len(req->src, req->src_len + req->dst_len), - buffer, req->src_len + req->dst_len, 0); - - ret = asn1_ber_decoder(&ecdsasignature_decoder, &sig_ctx, - buffer, req->src_len); - if (ret < 0) - goto error; - - if (bufsize > req->dst_len) - bufsize = req->dst_len; - - ecc_digits_from_bytes(buffer + req->src_len, bufsize, - hash, ctx->curve->g.ndigits); + if (slen != sizeof(*sig)) + return -EINVAL; - ret = _ecdsa_verify(ctx, hash, sig_ctx.r, sig_ctx.s); + if (bufsize > dlen) + bufsize = dlen; -error: - kfree(buffer); + ecc_digits_from_bytes(digest, bufsize, hash, ctx->curve->g.ndigits); - return ret; + return _ecdsa_verify(ctx, hash, sig->r, sig->s); } static int ecdsa_ecc_ctx_init(struct ecc_ctx *ctx, unsigned int curve_id) @@ -201,9 +119,10 @@ static int ecdsa_ecc_ctx_reset(struct ecc_ctx *ctx) * Set the public ECC key as defined by RFC5480 section 2.2 "Subject Public * Key". Only the uncompressed format is supported. */ -static int ecdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) +static int ecdsa_set_pub_key(struct crypto_sig *tfm, const void *key, + unsigned int keylen) { - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); unsigned int digitlen, ndigits; const unsigned char *d = key; int ret; @@ -237,31 +156,43 @@ static int ecdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, unsig return ret; } -static void ecdsa_exit_tfm(struct crypto_akcipher *tfm) +static void ecdsa_exit_tfm(struct crypto_sig *tfm) { - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); ecdsa_ecc_ctx_deinit(ctx); } -static unsigned int ecdsa_max_size(struct crypto_akcipher *tfm) +static unsigned int ecdsa_key_size(struct crypto_sig *tfm) { - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); return DIV_ROUND_UP(ctx->curve->nbits, 8); } -static int ecdsa_nist_p521_init_tfm(struct crypto_akcipher *tfm) +static unsigned int ecdsa_digest_size(struct crypto_sig *tfm) { - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + /* + * ECDSA key sizes are much smaller than RSA, and thus could + * operate on (hashed) inputs that are larger than the key size. + * E.g. SHA384-hashed input used with secp256r1 based keys. + * Return the largest supported hash size (SHA512). + */ + return SHA512_DIGEST_SIZE; +} + +static int ecdsa_nist_p521_init_tfm(struct crypto_sig *tfm) +{ + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); return ecdsa_ecc_ctx_init(ctx, ECC_CURVE_NIST_P521); } -static struct akcipher_alg ecdsa_nist_p521 = { +static struct sig_alg ecdsa_nist_p521 = { .verify = ecdsa_verify, .set_pub_key = ecdsa_set_pub_key, - .max_size = ecdsa_max_size, + .key_size = ecdsa_key_size, + .digest_size = ecdsa_digest_size, .init = ecdsa_nist_p521_init_tfm, .exit = ecdsa_exit_tfm, .base = { @@ -273,17 +204,18 @@ static struct akcipher_alg ecdsa_nist_p521 = { }, }; -static int ecdsa_nist_p384_init_tfm(struct crypto_akcipher *tfm) +static int ecdsa_nist_p384_init_tfm(struct crypto_sig *tfm) { - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); return ecdsa_ecc_ctx_init(ctx, ECC_CURVE_NIST_P384); } -static struct akcipher_alg ecdsa_nist_p384 = { +static struct sig_alg ecdsa_nist_p384 = { .verify = ecdsa_verify, .set_pub_key = ecdsa_set_pub_key, - .max_size = ecdsa_max_size, + .key_size = ecdsa_key_size, + .digest_size = ecdsa_digest_size, .init = ecdsa_nist_p384_init_tfm, .exit = ecdsa_exit_tfm, .base = { @@ -295,17 +227,18 @@ static struct akcipher_alg ecdsa_nist_p384 = { }, }; -static int ecdsa_nist_p256_init_tfm(struct crypto_akcipher *tfm) +static int ecdsa_nist_p256_init_tfm(struct crypto_sig *tfm) { - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); return ecdsa_ecc_ctx_init(ctx, ECC_CURVE_NIST_P256); } -static struct akcipher_alg ecdsa_nist_p256 = { +static struct sig_alg ecdsa_nist_p256 = { .verify = ecdsa_verify, .set_pub_key = ecdsa_set_pub_key, - .max_size = ecdsa_max_size, + .key_size = ecdsa_key_size, + .digest_size = ecdsa_digest_size, .init = ecdsa_nist_p256_init_tfm, .exit = ecdsa_exit_tfm, .base = { @@ -317,17 +250,18 @@ static struct akcipher_alg ecdsa_nist_p256 = { }, }; -static int ecdsa_nist_p192_init_tfm(struct crypto_akcipher *tfm) +static int ecdsa_nist_p192_init_tfm(struct crypto_sig *tfm) { - struct ecc_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecc_ctx *ctx = crypto_sig_ctx(tfm); return ecdsa_ecc_ctx_init(ctx, ECC_CURVE_NIST_P192); } -static struct akcipher_alg ecdsa_nist_p192 = { +static struct sig_alg ecdsa_nist_p192 = { .verify = ecdsa_verify, .set_pub_key = ecdsa_set_pub_key, - .max_size = ecdsa_max_size, + .key_size = ecdsa_key_size, + .digest_size = ecdsa_digest_size, .init = ecdsa_nist_p192_init_tfm, .exit = ecdsa_exit_tfm, .base = { @@ -345,42 +279,59 @@ static int __init ecdsa_init(void) int ret; /* NIST p192 may not be available in FIPS mode */ - ret = crypto_register_akcipher(&ecdsa_nist_p192); + ret = crypto_register_sig(&ecdsa_nist_p192); ecdsa_nist_p192_registered = ret == 0; - ret = crypto_register_akcipher(&ecdsa_nist_p256); + ret = crypto_register_sig(&ecdsa_nist_p256); if (ret) goto nist_p256_error; - ret = crypto_register_akcipher(&ecdsa_nist_p384); + ret = crypto_register_sig(&ecdsa_nist_p384); if (ret) goto nist_p384_error; - ret = crypto_register_akcipher(&ecdsa_nist_p521); + ret = crypto_register_sig(&ecdsa_nist_p521); if (ret) goto nist_p521_error; + ret = crypto_register_template(&ecdsa_x962_tmpl); + if (ret) + goto x962_tmpl_error; + + ret = crypto_register_template(&ecdsa_p1363_tmpl); + if (ret) + goto p1363_tmpl_error; + return 0; +p1363_tmpl_error: + crypto_unregister_template(&ecdsa_x962_tmpl); + +x962_tmpl_error: + crypto_unregister_sig(&ecdsa_nist_p521); + nist_p521_error: - crypto_unregister_akcipher(&ecdsa_nist_p384); + crypto_unregister_sig(&ecdsa_nist_p384); nist_p384_error: - crypto_unregister_akcipher(&ecdsa_nist_p256); + crypto_unregister_sig(&ecdsa_nist_p256); nist_p256_error: if (ecdsa_nist_p192_registered) - crypto_unregister_akcipher(&ecdsa_nist_p192); + crypto_unregister_sig(&ecdsa_nist_p192); return ret; } static void __exit ecdsa_exit(void) { + crypto_unregister_template(&ecdsa_x962_tmpl); + crypto_unregister_template(&ecdsa_p1363_tmpl); + if (ecdsa_nist_p192_registered) - crypto_unregister_akcipher(&ecdsa_nist_p192); - crypto_unregister_akcipher(&ecdsa_nist_p256); - crypto_unregister_akcipher(&ecdsa_nist_p384); - crypto_unregister_akcipher(&ecdsa_nist_p521); + crypto_unregister_sig(&ecdsa_nist_p192); + crypto_unregister_sig(&ecdsa_nist_p256); + crypto_unregister_sig(&ecdsa_nist_p384); + crypto_unregister_sig(&ecdsa_nist_p521); } subsys_initcall(ecdsa_init); diff --git a/crypto/ecrdsa.c b/crypto/ecrdsa.c index 3811f3805b5d..b3dd8a3ddeb7 100644 --- a/crypto/ecrdsa.c +++ b/crypto/ecrdsa.c @@ -18,12 +18,11 @@ #include <linux/module.h> #include <linux/crypto.h> +#include <crypto/sig.h> #include <crypto/streebog.h> -#include <crypto/internal/akcipher.h> #include <crypto/internal/ecc.h> -#include <crypto/akcipher.h> +#include <crypto/internal/sig.h> #include <linux/oid_registry.h> -#include <linux/scatterlist.h> #include "ecrdsa_params.asn1.h" #include "ecrdsa_pub_key.asn1.h" #include "ecrdsa_defs.h" @@ -68,13 +67,12 @@ static const struct ecc_curve *get_curve_by_oid(enum OID oid) } } -static int ecrdsa_verify(struct akcipher_request *req) +static int ecrdsa_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) { - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); - unsigned char sig[ECRDSA_MAX_SIG_SIZE]; - unsigned char digest[STREEBOG512_DIGEST_SIZE]; - unsigned int ndigits = req->dst_len / sizeof(u64); + struct ecrdsa_ctx *ctx = crypto_sig_ctx(tfm); + unsigned int ndigits = dlen / sizeof(u64); u64 r[ECRDSA_MAX_DIGITS]; /* witness (r) */ u64 _r[ECRDSA_MAX_DIGITS]; /* -r */ u64 s[ECRDSA_MAX_DIGITS]; /* second part of sig (s) */ @@ -91,25 +89,19 @@ static int ecrdsa_verify(struct akcipher_request *req) */ if (!ctx->curve || !ctx->digest || - !req->src || + !src || + !digest || !ctx->pub_key.x || - req->dst_len != ctx->digest_len || - req->dst_len != ctx->curve->g.ndigits * sizeof(u64) || + dlen != ctx->digest_len || + dlen != ctx->curve->g.ndigits * sizeof(u64) || ctx->pub_key.ndigits != ctx->curve->g.ndigits || - req->dst_len * 2 != req->src_len || - WARN_ON(req->src_len > sizeof(sig)) || - WARN_ON(req->dst_len > sizeof(digest))) + dlen * 2 != slen || + WARN_ON(slen > ECRDSA_MAX_SIG_SIZE) || + WARN_ON(dlen > STREEBOG512_DIGEST_SIZE)) return -EBADMSG; - sg_copy_to_buffer(req->src, sg_nents_for_len(req->src, req->src_len), - sig, req->src_len); - sg_pcopy_to_buffer(req->src, - sg_nents_for_len(req->src, - req->src_len + req->dst_len), - digest, req->dst_len, req->src_len); - - vli_from_be64(s, sig, ndigits); - vli_from_be64(r, sig + ndigits * sizeof(u64), ndigits); + vli_from_be64(s, src, ndigits); + vli_from_be64(r, src + ndigits * sizeof(u64), ndigits); /* Step 1: verify that 0 < r < q, 0 < s < q */ if (vli_is_zero(r, ndigits) || @@ -188,10 +180,10 @@ static u8 *ecrdsa_unpack_u32(u32 *dst, void *src) } /* Parse BER encoded subjectPublicKey. */ -static int ecrdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, +static int ecrdsa_set_pub_key(struct crypto_sig *tfm, const void *key, unsigned int keylen) { - struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecrdsa_ctx *ctx = crypto_sig_ctx(tfm); unsigned int ndigits; u32 algo, paramlen; u8 *params; @@ -249,9 +241,9 @@ static int ecrdsa_set_pub_key(struct crypto_akcipher *tfm, const void *key, return 0; } -static unsigned int ecrdsa_max_size(struct crypto_akcipher *tfm) +static unsigned int ecrdsa_key_size(struct crypto_sig *tfm) { - struct ecrdsa_ctx *ctx = akcipher_tfm_ctx(tfm); + struct ecrdsa_ctx *ctx = crypto_sig_ctx(tfm); /* * Verify doesn't need any output, so it's just informational @@ -260,13 +252,21 @@ static unsigned int ecrdsa_max_size(struct crypto_akcipher *tfm) return ctx->pub_key.ndigits * sizeof(u64); } -static void ecrdsa_exit_tfm(struct crypto_akcipher *tfm) +static unsigned int ecrdsa_max_size(struct crypto_sig *tfm) +{ + struct ecrdsa_ctx *ctx = crypto_sig_ctx(tfm); + + return 2 * ctx->pub_key.ndigits * sizeof(u64); +} + +static void ecrdsa_exit_tfm(struct crypto_sig *tfm) { } -static struct akcipher_alg ecrdsa_alg = { +static struct sig_alg ecrdsa_alg = { .verify = ecrdsa_verify, .set_pub_key = ecrdsa_set_pub_key, + .key_size = ecrdsa_key_size, .max_size = ecrdsa_max_size, .exit = ecrdsa_exit_tfm, .base = { @@ -280,12 +280,12 @@ static struct akcipher_alg ecrdsa_alg = { static int __init ecrdsa_mod_init(void) { - return crypto_register_akcipher(&ecrdsa_alg); + return crypto_register_sig(&ecrdsa_alg); } static void __exit ecrdsa_mod_fini(void) { - crypto_unregister_akcipher(&ecrdsa_alg); + crypto_unregister_sig(&ecrdsa_alg); } module_init(ecrdsa_mod_init); diff --git a/crypto/internal.h b/crypto/internal.h index 711a6a5bfa2b..46b661be0f90 100644 --- a/crypto/internal.h +++ b/crypto/internal.h @@ -22,8 +22,6 @@ #include <linux/sched.h> #include <linux/types.h> -struct akcipher_request; -struct crypto_akcipher; struct crypto_instance; struct crypto_template; @@ -35,19 +33,6 @@ struct crypto_larval { bool test_started; }; -struct crypto_akcipher_sync_data { - struct crypto_akcipher *tfm; - const void *src; - void *dst; - unsigned int slen; - unsigned int dlen; - - struct akcipher_request *req; - struct crypto_wait cwait; - struct scatterlist sg; - u8 *buf; -}; - enum { CRYPTOA_UNSPEC, CRYPTOA_ALG, @@ -129,10 +114,6 @@ void *crypto_create_tfm_node(struct crypto_alg *alg, void *crypto_clone_tfm(const struct crypto_type *frontend, struct crypto_tfm *otfm); -int crypto_akcipher_sync_prep(struct crypto_akcipher_sync_data *data); -int crypto_akcipher_sync_post(struct crypto_akcipher_sync_data *data, int err); -int crypto_init_akcipher_ops_sig(struct crypto_tfm *tfm); - static inline void *crypto_create_tfm(struct crypto_alg *alg, const struct crypto_type *frontend) { diff --git a/crypto/jitterentropy-testing.c b/crypto/jitterentropy-testing.c index 5cb6a77b8e3b..21c9d7c3269a 100644 --- a/crypto/jitterentropy-testing.c +++ b/crypto/jitterentropy-testing.c @@ -15,7 +15,7 @@ #define JENT_TEST_RINGBUFFER_MASK (JENT_TEST_RINGBUFFER_SIZE - 1) struct jent_testing { - u32 jent_testing_rb[JENT_TEST_RINGBUFFER_SIZE]; + u64 jent_testing_rb[JENT_TEST_RINGBUFFER_SIZE]; u32 rb_reader; atomic_t rb_writer; atomic_t jent_testing_enabled; @@ -72,7 +72,7 @@ static void jent_testing_fini(struct jent_testing *data, u32 boot) pr_warn("Disabling data collection\n"); } -static bool jent_testing_store(struct jent_testing *data, u32 value, +static bool jent_testing_store(struct jent_testing *data, u64 value, u32 *boot) { unsigned long flags; @@ -156,20 +156,20 @@ static int jent_testing_reader(struct jent_testing *data, u32 *boot, } /* We copy out word-wise */ - if (outbuflen < sizeof(u32)) { + if (outbuflen < sizeof(u64)) { spin_unlock_irqrestore(&data->lock, flags); goto out; } memcpy(outbuf, &data->jent_testing_rb[data->rb_reader], - sizeof(u32)); + sizeof(u64)); data->rb_reader++; spin_unlock_irqrestore(&data->lock, flags); - outbuf += sizeof(u32); - outbuflen -= sizeof(u32); - collected_data += sizeof(u32); + outbuf += sizeof(u64); + outbuflen -= sizeof(u64); + collected_data += sizeof(u64); } out: @@ -189,16 +189,17 @@ static int jent_testing_extract_user(struct file *file, char __user *buf, /* * The intention of this interface is for collecting at least - * 1000 samples due to the SP800-90B requirements. So, we make no - * effort in avoiding allocating more memory that actually needed - * by the user. Hence, we allocate sufficient memory to always hold - * that amount of data. + * 1000 samples due to the SP800-90B requirements. However, due to + * memory and performance constraints, it is not desirable to allocate + * 8000 bytes of memory. Instead, we allocate space for only 125 + * samples, which will allow the user to collect all 1000 samples using + * 8 calls to this interface. */ - tmp = kmalloc(JENT_TEST_RINGBUFFER_SIZE + sizeof(u32), GFP_KERNEL); + tmp = kmalloc(125 * sizeof(u64) + sizeof(u64), GFP_KERNEL); if (!tmp) return -ENOMEM; - tmp_aligned = PTR_ALIGN(tmp, sizeof(u32)); + tmp_aligned = PTR_ALIGN(tmp, sizeof(u64)); while (nbytes) { int i; @@ -212,7 +213,7 @@ static int jent_testing_extract_user(struct file *file, char __user *buf, schedule(); } - i = min_t(int, nbytes, JENT_TEST_RINGBUFFER_SIZE); + i = min_t(int, nbytes, 125 * sizeof(u64)); i = reader(tmp_aligned, i); if (i <= 0) { if (i < 0) @@ -251,7 +252,7 @@ static struct jent_testing jent_raw_hires = { .read_wait = __WAIT_QUEUE_HEAD_INITIALIZER(jent_raw_hires.read_wait) }; -int jent_raw_hires_entropy_store(__u32 value) +int jent_raw_hires_entropy_store(__u64 value) { return jent_testing_store(&jent_raw_hires, value, &boot_raw_hires_test); } diff --git a/crypto/jitterentropy.h b/crypto/jitterentropy.h index aa4728675ae2..4c5dbf2a8d8f 100644 --- a/crypto/jitterentropy.h +++ b/crypto/jitterentropy.h @@ -22,11 +22,11 @@ extern struct rand_data *jent_entropy_collector_alloc(unsigned int osr, extern void jent_entropy_collector_free(struct rand_data *entropy_collector); #ifdef CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE -int jent_raw_hires_entropy_store(__u32 value); +int jent_raw_hires_entropy_store(__u64 value); void jent_testing_init(void); void jent_testing_exit(void); #else /* CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE */ -static inline int jent_raw_hires_entropy_store(__u32 value) { return 0; } +static inline int jent_raw_hires_entropy_store(__u64 value) { return 0; } static inline void jent_testing_init(void) { } static inline void jent_testing_exit(void) { } #endif /* CONFIG_CRYPTO_JITTERENTROPY_TESTINTERFACE */ diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index d0d954fe9d54..7fc79e7dce44 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -117,8 +117,10 @@ static int pcrypt_aead_encrypt(struct aead_request *req) err = padata_do_parallel(ictx->psenc, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; - if (err == -EBUSY) - return -EAGAIN; + if (err == -EBUSY) { + /* try non-parallel mode */ + return crypto_aead_encrypt(creq); + } return err; } @@ -166,8 +168,10 @@ static int pcrypt_aead_decrypt(struct aead_request *req) err = padata_do_parallel(ictx->psdec, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; - if (err == -EBUSY) - return -EAGAIN; + if (err == -EBUSY) { + /* try non-parallel mode */ + return crypto_aead_decrypt(creq); + } return err; } diff --git a/crypto/rsa-pkcs1pad.c b/crypto/rsa-pkcs1pad.c index cd501195f34a..50bdb18e7b48 100644 --- a/crypto/rsa-pkcs1pad.c +++ b/crypto/rsa-pkcs1pad.c @@ -16,101 +16,6 @@ #include <linux/random.h> #include <linux/scatterlist.h> -/* - * Hash algorithm OIDs plus ASN.1 DER wrappings [RFC4880 sec 5.2.2]. - */ -static const u8 rsa_digest_info_md5[] = { - 0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, - 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, /* OID */ - 0x05, 0x00, 0x04, 0x10 -}; - -static const u8 rsa_digest_info_sha1[] = { - 0x30, 0x21, 0x30, 0x09, 0x06, 0x05, - 0x2b, 0x0e, 0x03, 0x02, 0x1a, - 0x05, 0x00, 0x04, 0x14 -}; - -static const u8 rsa_digest_info_rmd160[] = { - 0x30, 0x21, 0x30, 0x09, 0x06, 0x05, - 0x2b, 0x24, 0x03, 0x02, 0x01, - 0x05, 0x00, 0x04, 0x14 -}; - -static const u8 rsa_digest_info_sha224[] = { - 0x30, 0x2d, 0x30, 0x0d, 0x06, 0x09, - 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x04, - 0x05, 0x00, 0x04, 0x1c -}; - -static const u8 rsa_digest_info_sha256[] = { - 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, - 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, - 0x05, 0x00, 0x04, 0x20 -}; - -static const u8 rsa_digest_info_sha384[] = { - 0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, - 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, - 0x05, 0x00, 0x04, 0x30 -}; - -static const u8 rsa_digest_info_sha512[] = { - 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, - 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, - 0x05, 0x00, 0x04, 0x40 -}; - -static const u8 rsa_digest_info_sha3_256[] = { - 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, - 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x08, - 0x05, 0x00, 0x04, 0x20 -}; - -static const u8 rsa_digest_info_sha3_384[] = { - 0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, - 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x09, - 0x05, 0x00, 0x04, 0x30 -}; - -static const u8 rsa_digest_info_sha3_512[] = { - 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, - 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x0A, - 0x05, 0x00, 0x04, 0x40 -}; - -static const struct rsa_asn1_template { - const char *name; - const u8 *data; - size_t size; -} rsa_asn1_templates[] = { -#define _(X) { #X, rsa_digest_info_##X, sizeof(rsa_digest_info_##X) } - _(md5), - _(sha1), - _(rmd160), - _(sha256), - _(sha384), - _(sha512), - _(sha224), -#undef _ -#define _(X) { "sha3-" #X, rsa_digest_info_sha3_##X, sizeof(rsa_digest_info_sha3_##X) } - _(256), - _(384), - _(512), -#undef _ - { NULL } -}; - -static const struct rsa_asn1_template *rsa_lookup_asn1(const char *name) -{ - const struct rsa_asn1_template *p; - - for (p = rsa_asn1_templates; p->name; p++) - if (strcmp(name, p->name) == 0) - return p; - return NULL; -} - struct pkcs1pad_ctx { struct crypto_akcipher *child; unsigned int key_size; @@ -118,7 +23,6 @@ struct pkcs1pad_ctx { struct pkcs1pad_inst_ctx { struct crypto_akcipher_spawn spawn; - const struct rsa_asn1_template *digest_info; }; struct pkcs1pad_request { @@ -131,42 +35,16 @@ static int pkcs1pad_set_pub_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); - int err; - - ctx->key_size = 0; - - err = crypto_akcipher_set_pub_key(ctx->child, key, keylen); - if (err) - return err; - - /* Find out new modulus size from rsa implementation */ - err = crypto_akcipher_maxsize(ctx->child); - if (err > PAGE_SIZE) - return -ENOTSUPP; - ctx->key_size = err; - return 0; + return rsa_set_key(ctx->child, &ctx->key_size, RSA_PUB, key, keylen); } static int pkcs1pad_set_priv_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen) { struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); - int err; - - ctx->key_size = 0; - - err = crypto_akcipher_set_priv_key(ctx->child, key, keylen); - if (err) - return err; - /* Find out new modulus size from rsa implementation */ - err = crypto_akcipher_maxsize(ctx->child); - if (err > PAGE_SIZE) - return -ENOTSUPP; - - ctx->key_size = err; - return 0; + return rsa_set_key(ctx->child, &ctx->key_size, RSA_PRIV, key, keylen); } static unsigned int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) @@ -174,9 +52,9 @@ static unsigned int pkcs1pad_get_max_size(struct crypto_akcipher *tfm) struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); /* - * The maximum destination buffer size for the encrypt/sign operations + * The maximum destination buffer size for the encrypt operation * will be the same as for RSA, even though it's smaller for - * decrypt/verify. + * decrypt. */ return ctx->key_size; @@ -194,7 +72,7 @@ static void pkcs1pad_sg_set_buf(struct scatterlist *sg, void *buf, size_t len, sg_chain(sg, nsegs, next); } -static int pkcs1pad_encrypt_sign_complete(struct akcipher_request *req, int err) +static int pkcs1pad_encrypt_complete(struct akcipher_request *req, int err) { struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); @@ -233,14 +111,14 @@ out: return err; } -static void pkcs1pad_encrypt_sign_complete_cb(void *data, int err) +static void pkcs1pad_encrypt_complete_cb(void *data, int err) { struct akcipher_request *req = data; if (err == -EINPROGRESS) goto out; - err = pkcs1pad_encrypt_sign_complete(req, err); + err = pkcs1pad_encrypt_complete(req, err); out: akcipher_request_complete(req, err); @@ -281,7 +159,7 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) akcipher_request_set_tfm(&req_ctx->child_req, ctx->child); akcipher_request_set_callback(&req_ctx->child_req, req->base.flags, - pkcs1pad_encrypt_sign_complete_cb, req); + pkcs1pad_encrypt_complete_cb, req); /* Reuse output buffer */ akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg, @@ -289,7 +167,7 @@ static int pkcs1pad_encrypt(struct akcipher_request *req) err = crypto_akcipher_encrypt(&req_ctx->child_req); if (err != -EINPROGRESS && err != -EBUSY) - return pkcs1pad_encrypt_sign_complete(req, err); + return pkcs1pad_encrypt_complete(req, err); return err; } @@ -394,195 +272,6 @@ static int pkcs1pad_decrypt(struct akcipher_request *req) return err; } -static int pkcs1pad_sign(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); - struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); - struct akcipher_instance *inst = akcipher_alg_instance(tfm); - struct pkcs1pad_inst_ctx *ictx = akcipher_instance_ctx(inst); - const struct rsa_asn1_template *digest_info = ictx->digest_info; - int err; - unsigned int ps_end, digest_info_size = 0; - - if (!ctx->key_size) - return -EINVAL; - - if (digest_info) - digest_info_size = digest_info->size; - - if (req->src_len + digest_info_size > ctx->key_size - 11) - return -EOVERFLOW; - - if (req->dst_len < ctx->key_size) { - req->dst_len = ctx->key_size; - return -EOVERFLOW; - } - - req_ctx->in_buf = kmalloc(ctx->key_size - 1 - req->src_len, - GFP_KERNEL); - if (!req_ctx->in_buf) - return -ENOMEM; - - ps_end = ctx->key_size - digest_info_size - req->src_len - 2; - req_ctx->in_buf[0] = 0x01; - memset(req_ctx->in_buf + 1, 0xff, ps_end - 1); - req_ctx->in_buf[ps_end] = 0x00; - - if (digest_info) - memcpy(req_ctx->in_buf + ps_end + 1, digest_info->data, - digest_info->size); - - pkcs1pad_sg_set_buf(req_ctx->in_sg, req_ctx->in_buf, - ctx->key_size - 1 - req->src_len, req->src); - - akcipher_request_set_tfm(&req_ctx->child_req, ctx->child); - akcipher_request_set_callback(&req_ctx->child_req, req->base.flags, - pkcs1pad_encrypt_sign_complete_cb, req); - - /* Reuse output buffer */ - akcipher_request_set_crypt(&req_ctx->child_req, req_ctx->in_sg, - req->dst, ctx->key_size - 1, req->dst_len); - - err = crypto_akcipher_decrypt(&req_ctx->child_req); - if (err != -EINPROGRESS && err != -EBUSY) - return pkcs1pad_encrypt_sign_complete(req, err); - - return err; -} - -static int pkcs1pad_verify_complete(struct akcipher_request *req, int err) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); - struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); - struct akcipher_instance *inst = akcipher_alg_instance(tfm); - struct pkcs1pad_inst_ctx *ictx = akcipher_instance_ctx(inst); - const struct rsa_asn1_template *digest_info = ictx->digest_info; - const unsigned int sig_size = req->src_len; - const unsigned int digest_size = req->dst_len; - unsigned int dst_len; - unsigned int pos; - u8 *out_buf; - - if (err) - goto done; - - err = -EINVAL; - dst_len = req_ctx->child_req.dst_len; - if (dst_len < ctx->key_size - 1) - goto done; - - out_buf = req_ctx->out_buf; - if (dst_len == ctx->key_size) { - if (out_buf[0] != 0x00) - /* Decrypted value had no leading 0 byte */ - goto done; - - dst_len--; - out_buf++; - } - - err = -EBADMSG; - if (out_buf[0] != 0x01) - goto done; - - for (pos = 1; pos < dst_len; pos++) - if (out_buf[pos] != 0xff) - break; - - if (pos < 9 || pos == dst_len || out_buf[pos] != 0x00) - goto done; - pos++; - - if (digest_info) { - if (digest_info->size > dst_len - pos) - goto done; - if (crypto_memneq(out_buf + pos, digest_info->data, - digest_info->size)) - goto done; - - pos += digest_info->size; - } - - err = 0; - - if (digest_size != dst_len - pos) { - err = -EKEYREJECTED; - req->dst_len = dst_len - pos; - goto done; - } - /* Extract appended digest. */ - sg_pcopy_to_buffer(req->src, - sg_nents_for_len(req->src, sig_size + digest_size), - req_ctx->out_buf + ctx->key_size, - digest_size, sig_size); - /* Do the actual verification step. */ - if (memcmp(req_ctx->out_buf + ctx->key_size, out_buf + pos, - digest_size) != 0) - err = -EKEYREJECTED; -done: - kfree_sensitive(req_ctx->out_buf); - - return err; -} - -static void pkcs1pad_verify_complete_cb(void *data, int err) -{ - struct akcipher_request *req = data; - - if (err == -EINPROGRESS) - goto out; - - err = pkcs1pad_verify_complete(req, err); - -out: - akcipher_request_complete(req, err); -} - -/* - * The verify operation is here for completeness similar to the verification - * defined in RFC2313 section 10.2 except that block type 0 is not accepted, - * as in RFC2437. RFC2437 section 9.2 doesn't define any operation to - * retrieve the DigestInfo from a signature, instead the user is expected - * to call the sign operation to generate the expected signature and compare - * signatures instead of the message-digests. - */ -static int pkcs1pad_verify(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - struct pkcs1pad_ctx *ctx = akcipher_tfm_ctx(tfm); - struct pkcs1pad_request *req_ctx = akcipher_request_ctx(req); - const unsigned int sig_size = req->src_len; - const unsigned int digest_size = req->dst_len; - int err; - - if (WARN_ON(req->dst) || WARN_ON(!digest_size) || - !ctx->key_size || sig_size != ctx->key_size) - return -EINVAL; - - req_ctx->out_buf = kmalloc(ctx->key_size + digest_size, GFP_KERNEL); - if (!req_ctx->out_buf) - return -ENOMEM; - - pkcs1pad_sg_set_buf(req_ctx->out_sg, req_ctx->out_buf, - ctx->key_size, NULL); - - akcipher_request_set_tfm(&req_ctx->child_req, ctx->child); - akcipher_request_set_callback(&req_ctx->child_req, req->base.flags, - pkcs1pad_verify_complete_cb, req); - - /* Reuse input buffer, output to a new buffer */ - akcipher_request_set_crypt(&req_ctx->child_req, req->src, - req_ctx->out_sg, sig_size, ctx->key_size); - - err = crypto_akcipher_encrypt(&req_ctx->child_req); - if (err != -EINPROGRESS && err != -EBUSY) - return pkcs1pad_verify_complete(req, err); - - return err; -} - static int pkcs1pad_init_tfm(struct crypto_akcipher *tfm) { struct akcipher_instance *inst = akcipher_alg_instance(tfm); @@ -624,7 +313,6 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) struct akcipher_instance *inst; struct pkcs1pad_inst_ctx *ctx; struct akcipher_alg *rsa_alg; - const char *hash_name; int err; err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_AKCIPHER, &mask); @@ -650,36 +338,15 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) } err = -ENAMETOOLONG; - hash_name = crypto_attr_alg_name(tb[2]); - if (IS_ERR(hash_name)) { - if (snprintf(inst->alg.base.cra_name, - CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)", - rsa_alg->base.cra_name) >= CRYPTO_MAX_ALG_NAME) - goto err_free_inst; - - if (snprintf(inst->alg.base.cra_driver_name, - CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)", - rsa_alg->base.cra_driver_name) >= - CRYPTO_MAX_ALG_NAME) - goto err_free_inst; - } else { - ctx->digest_info = rsa_lookup_asn1(hash_name); - if (!ctx->digest_info) { - err = -EINVAL; - goto err_free_inst; - } - - if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, - "pkcs1pad(%s,%s)", rsa_alg->base.cra_name, - hash_name) >= CRYPTO_MAX_ALG_NAME) - goto err_free_inst; - - if (snprintf(inst->alg.base.cra_driver_name, - CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s,%s)", - rsa_alg->base.cra_driver_name, - hash_name) >= CRYPTO_MAX_ALG_NAME) - goto err_free_inst; - } + if (snprintf(inst->alg.base.cra_name, + CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)", + rsa_alg->base.cra_name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + + if (snprintf(inst->alg.base.cra_driver_name, + CRYPTO_MAX_ALG_NAME, "pkcs1pad(%s)", + rsa_alg->base.cra_driver_name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; inst->alg.base.cra_priority = rsa_alg->base.cra_priority; inst->alg.base.cra_ctxsize = sizeof(struct pkcs1pad_ctx); @@ -689,8 +356,6 @@ static int pkcs1pad_create(struct crypto_template *tmpl, struct rtattr **tb) inst->alg.encrypt = pkcs1pad_encrypt; inst->alg.decrypt = pkcs1pad_decrypt; - inst->alg.sign = pkcs1pad_sign; - inst->alg.verify = pkcs1pad_verify; inst->alg.set_pub_key = pkcs1pad_set_pub_key; inst->alg.set_priv_key = pkcs1pad_set_priv_key; inst->alg.max_size = pkcs1pad_get_max_size; diff --git a/crypto/rsa.c b/crypto/rsa.c index 78b28d14ced3..b7d21529c552 100644 --- a/crypto/rsa.c +++ b/crypto/rsa.c @@ -407,16 +407,25 @@ static int __init rsa_init(void) return err; err = crypto_register_template(&rsa_pkcs1pad_tmpl); - if (err) { - crypto_unregister_akcipher(&rsa); - return err; - } + if (err) + goto err_unregister_rsa; + + err = crypto_register_template(&rsassa_pkcs1_tmpl); + if (err) + goto err_unregister_rsa_pkcs1pad; return 0; + +err_unregister_rsa_pkcs1pad: + crypto_unregister_template(&rsa_pkcs1pad_tmpl); +err_unregister_rsa: + crypto_unregister_akcipher(&rsa); + return err; } static void __exit rsa_exit(void) { + crypto_unregister_template(&rsassa_pkcs1_tmpl); crypto_unregister_template(&rsa_pkcs1pad_tmpl); crypto_unregister_akcipher(&rsa); } diff --git a/crypto/rsassa-pkcs1.c b/crypto/rsassa-pkcs1.c new file mode 100644 index 000000000000..4d077fc96076 --- /dev/null +++ b/crypto/rsassa-pkcs1.c @@ -0,0 +1,454 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * RSA Signature Scheme with Appendix - PKCS #1 v1.5 (RFC 8017 sec 8.2) + * + * https://www.rfc-editor.org/rfc/rfc8017#section-8.2 + * + * Copyright (c) 2015 - 2024 Intel Corporation + */ + +#include <linux/module.h> +#include <linux/scatterlist.h> +#include <crypto/akcipher.h> +#include <crypto/algapi.h> +#include <crypto/hash.h> +#include <crypto/sig.h> +#include <crypto/internal/akcipher.h> +#include <crypto/internal/rsa.h> +#include <crypto/internal/sig.h> + +/* + * Full Hash Prefix for EMSA-PKCS1-v1_5 encoding method (RFC 9580 table 24) + * + * RSA keys are usually much larger than the hash of the message to be signed. + * The hash is therefore prepended by the Full Hash Prefix and a 0xff padding. + * The Full Hash Prefix is an ASN.1 SEQUENCE containing the hash algorithm OID. + * + * https://www.rfc-editor.org/rfc/rfc9580#table-24 + */ + +static const u8 hash_prefix_none[] = { }; + +static const u8 hash_prefix_md5[] = { + 0x30, 0x20, 0x30, 0x0c, 0x06, 0x08, /* SEQUENCE (SEQUENCE (OID */ + 0x2a, 0x86, 0x48, 0x86, 0xf7, 0x0d, 0x02, 0x05, /* <algorithm>, */ + 0x05, 0x00, 0x04, 0x10 /* NULL), OCTET STRING <hash>) */ +}; + +static const u8 hash_prefix_sha1[] = { + 0x30, 0x21, 0x30, 0x09, 0x06, 0x05, + 0x2b, 0x0e, 0x03, 0x02, 0x1a, + 0x05, 0x00, 0x04, 0x14 +}; + +static const u8 hash_prefix_rmd160[] = { + 0x30, 0x21, 0x30, 0x09, 0x06, 0x05, + 0x2b, 0x24, 0x03, 0x02, 0x01, + 0x05, 0x00, 0x04, 0x14 +}; + +static const u8 hash_prefix_sha224[] = { + 0x30, 0x2d, 0x30, 0x0d, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x04, + 0x05, 0x00, 0x04, 0x1c +}; + +static const u8 hash_prefix_sha256[] = { + 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x01, + 0x05, 0x00, 0x04, 0x20 +}; + +static const u8 hash_prefix_sha384[] = { + 0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x02, + 0x05, 0x00, 0x04, 0x30 +}; + +static const u8 hash_prefix_sha512[] = { + 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x03, + 0x05, 0x00, 0x04, 0x40 +}; + +static const u8 hash_prefix_sha3_256[] = { + 0x30, 0x31, 0x30, 0x0d, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x08, + 0x05, 0x00, 0x04, 0x20 +}; + +static const u8 hash_prefix_sha3_384[] = { + 0x30, 0x41, 0x30, 0x0d, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x09, + 0x05, 0x00, 0x04, 0x30 +}; + +static const u8 hash_prefix_sha3_512[] = { + 0x30, 0x51, 0x30, 0x0d, 0x06, 0x09, + 0x60, 0x86, 0x48, 0x01, 0x65, 0x03, 0x04, 0x02, 0x0a, + 0x05, 0x00, 0x04, 0x40 +}; + +static const struct hash_prefix { + const char *name; + const u8 *data; + size_t size; +} hash_prefixes[] = { +#define _(X) { #X, hash_prefix_##X, sizeof(hash_prefix_##X) } + _(none), + _(md5), + _(sha1), + _(rmd160), + _(sha256), + _(sha384), + _(sha512), + _(sha224), +#undef _ +#define _(X) { "sha3-" #X, hash_prefix_sha3_##X, sizeof(hash_prefix_sha3_##X) } + _(256), + _(384), + _(512), +#undef _ + { NULL } +}; + +static const struct hash_prefix *rsassa_pkcs1_find_hash_prefix(const char *name) +{ + const struct hash_prefix *p; + + for (p = hash_prefixes; p->name; p++) + if (strcmp(name, p->name) == 0) + return p; + return NULL; +} + +static bool rsassa_pkcs1_invalid_hash_len(unsigned int len, + const struct hash_prefix *p) +{ + /* + * Legacy protocols such as TLS 1.1 or earlier and IKE version 1 + * do not prepend a Full Hash Prefix to the hash. In that case, + * the size of the Full Hash Prefix is zero. + */ + if (p->data == hash_prefix_none) + return false; + + /* + * The final byte of the Full Hash Prefix encodes the hash length. + * + * This needs to be revisited should hash algorithms with more than + * 1016 bits (127 bytes * 8) ever be added. The length would then + * be encoded into more than one byte by ASN.1. + */ + static_assert(HASH_MAX_DIGESTSIZE <= 127); + + return len != p->data[p->size - 1]; +} + +struct rsassa_pkcs1_ctx { + struct crypto_akcipher *child; + unsigned int key_size; +}; + +struct rsassa_pkcs1_inst_ctx { + struct crypto_akcipher_spawn spawn; + const struct hash_prefix *hash_prefix; +}; + +static int rsassa_pkcs1_sign(struct crypto_sig *tfm, + const void *src, unsigned int slen, + void *dst, unsigned int dlen) +{ + struct sig_instance *inst = sig_alg_instance(tfm); + struct rsassa_pkcs1_inst_ctx *ictx = sig_instance_ctx(inst); + const struct hash_prefix *hash_prefix = ictx->hash_prefix; + struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); + unsigned int child_reqsize = crypto_akcipher_reqsize(ctx->child); + struct akcipher_request *child_req __free(kfree_sensitive) = NULL; + struct scatterlist in_sg[3], out_sg; + struct crypto_wait cwait; + unsigned int pad_len; + unsigned int ps_end; + unsigned int len; + u8 *in_buf; + int err; + + if (!ctx->key_size) + return -EINVAL; + + if (dlen < ctx->key_size) + return -EOVERFLOW; + + if (rsassa_pkcs1_invalid_hash_len(slen, hash_prefix)) + return -EINVAL; + + if (slen + hash_prefix->size > ctx->key_size - 11) + return -EOVERFLOW; + + pad_len = ctx->key_size - slen - hash_prefix->size - 1; + + child_req = kmalloc(sizeof(*child_req) + child_reqsize + pad_len, + GFP_KERNEL); + if (!child_req) + return -ENOMEM; + + /* RFC 8017 sec 8.2.1 step 1 - EMSA-PKCS1-v1_5 encoding generation */ + in_buf = (u8 *)(child_req + 1) + child_reqsize; + ps_end = pad_len - 1; + in_buf[0] = 0x01; + memset(in_buf + 1, 0xff, ps_end - 1); + in_buf[ps_end] = 0x00; + + /* RFC 8017 sec 8.2.1 step 2 - RSA signature */ + crypto_init_wait(&cwait); + sg_init_table(in_sg, 3); + sg_set_buf(&in_sg[0], in_buf, pad_len); + sg_set_buf(&in_sg[1], hash_prefix->data, hash_prefix->size); + sg_set_buf(&in_sg[2], src, slen); + sg_init_one(&out_sg, dst, dlen); + akcipher_request_set_tfm(child_req, ctx->child); + akcipher_request_set_crypt(child_req, in_sg, &out_sg, + ctx->key_size - 1, dlen); + akcipher_request_set_callback(child_req, CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &cwait); + + err = crypto_akcipher_decrypt(child_req); + err = crypto_wait_req(err, &cwait); + if (err) + return err; + + len = child_req->dst_len; + pad_len = ctx->key_size - len; + + /* Four billion to one */ + if (unlikely(pad_len)) { + memmove(dst + pad_len, dst, len); + memset(dst, 0, pad_len); + } + + return 0; +} + +static int rsassa_pkcs1_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct sig_instance *inst = sig_alg_instance(tfm); + struct rsassa_pkcs1_inst_ctx *ictx = sig_instance_ctx(inst); + const struct hash_prefix *hash_prefix = ictx->hash_prefix; + struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); + unsigned int child_reqsize = crypto_akcipher_reqsize(ctx->child); + struct akcipher_request *child_req __free(kfree_sensitive) = NULL; + struct scatterlist in_sg, out_sg; + struct crypto_wait cwait; + unsigned int dst_len; + unsigned int pos; + u8 *out_buf; + int err; + + /* RFC 8017 sec 8.2.2 step 1 - length checking */ + if (!ctx->key_size || + slen != ctx->key_size || + rsassa_pkcs1_invalid_hash_len(dlen, hash_prefix)) + return -EINVAL; + + /* RFC 8017 sec 8.2.2 step 2 - RSA verification */ + child_req = kmalloc(sizeof(*child_req) + child_reqsize + ctx->key_size, + GFP_KERNEL); + if (!child_req) + return -ENOMEM; + + out_buf = (u8 *)(child_req + 1) + child_reqsize; + + crypto_init_wait(&cwait); + sg_init_one(&in_sg, src, slen); + sg_init_one(&out_sg, out_buf, ctx->key_size); + akcipher_request_set_tfm(child_req, ctx->child); + akcipher_request_set_crypt(child_req, &in_sg, &out_sg, + slen, ctx->key_size); + akcipher_request_set_callback(child_req, CRYPTO_TFM_REQ_MAY_SLEEP, + crypto_req_done, &cwait); + + err = crypto_akcipher_encrypt(child_req); + err = crypto_wait_req(err, &cwait); + if (err) + return err; + + /* RFC 8017 sec 8.2.2 step 3 - EMSA-PKCS1-v1_5 encoding verification */ + dst_len = child_req->dst_len; + if (dst_len < ctx->key_size - 1) + return -EINVAL; + + if (dst_len == ctx->key_size) { + if (out_buf[0] != 0x00) + /* Encrypted value had no leading 0 byte */ + return -EINVAL; + + dst_len--; + out_buf++; + } + + if (out_buf[0] != 0x01) + return -EBADMSG; + + for (pos = 1; pos < dst_len; pos++) + if (out_buf[pos] != 0xff) + break; + + if (pos < 9 || pos == dst_len || out_buf[pos] != 0x00) + return -EBADMSG; + pos++; + + if (hash_prefix->size > dst_len - pos) + return -EBADMSG; + if (crypto_memneq(out_buf + pos, hash_prefix->data, hash_prefix->size)) + return -EBADMSG; + pos += hash_prefix->size; + + /* RFC 8017 sec 8.2.2 step 4 - comparison of digest with out_buf */ + if (dlen != dst_len - pos) + return -EKEYREJECTED; + if (memcmp(digest, out_buf + pos, dlen) != 0) + return -EKEYREJECTED; + + return 0; +} + +static unsigned int rsassa_pkcs1_key_size(struct crypto_sig *tfm) +{ + struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); + + return ctx->key_size; +} + +static int rsassa_pkcs1_set_pub_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); + + return rsa_set_key(ctx->child, &ctx->key_size, RSA_PUB, key, keylen); +} + +static int rsassa_pkcs1_set_priv_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); + + return rsa_set_key(ctx->child, &ctx->key_size, RSA_PRIV, key, keylen); +} + +static int rsassa_pkcs1_init_tfm(struct crypto_sig *tfm) +{ + struct sig_instance *inst = sig_alg_instance(tfm); + struct rsassa_pkcs1_inst_ctx *ictx = sig_instance_ctx(inst); + struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); + struct crypto_akcipher *child_tfm; + + child_tfm = crypto_spawn_akcipher(&ictx->spawn); + if (IS_ERR(child_tfm)) + return PTR_ERR(child_tfm); + + ctx->child = child_tfm; + + return 0; +} + +static void rsassa_pkcs1_exit_tfm(struct crypto_sig *tfm) +{ + struct rsassa_pkcs1_ctx *ctx = crypto_sig_ctx(tfm); + + crypto_free_akcipher(ctx->child); +} + +static void rsassa_pkcs1_free(struct sig_instance *inst) +{ + struct rsassa_pkcs1_inst_ctx *ctx = sig_instance_ctx(inst); + struct crypto_akcipher_spawn *spawn = &ctx->spawn; + + crypto_drop_akcipher(spawn); + kfree(inst); +} + +static int rsassa_pkcs1_create(struct crypto_template *tmpl, struct rtattr **tb) +{ + struct rsassa_pkcs1_inst_ctx *ctx; + struct akcipher_alg *rsa_alg; + struct sig_instance *inst; + const char *hash_name; + u32 mask; + int err; + + err = crypto_check_attr_type(tb, CRYPTO_ALG_TYPE_SIG, &mask); + if (err) + return err; + + inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL); + if (!inst) + return -ENOMEM; + + ctx = sig_instance_ctx(inst); + + err = crypto_grab_akcipher(&ctx->spawn, sig_crypto_instance(inst), + crypto_attr_alg_name(tb[1]), 0, mask); + if (err) + goto err_free_inst; + + rsa_alg = crypto_spawn_akcipher_alg(&ctx->spawn); + + if (strcmp(rsa_alg->base.cra_name, "rsa") != 0) { + err = -EINVAL; + goto err_free_inst; + } + + hash_name = crypto_attr_alg_name(tb[2]); + if (IS_ERR(hash_name)) { + err = PTR_ERR(hash_name); + goto err_free_inst; + } + + ctx->hash_prefix = rsassa_pkcs1_find_hash_prefix(hash_name); + if (!ctx->hash_prefix) { + err = -EINVAL; + goto err_free_inst; + } + + err = -ENAMETOOLONG; + if (snprintf(inst->alg.base.cra_name, CRYPTO_MAX_ALG_NAME, + "pkcs1(%s,%s)", rsa_alg->base.cra_name, + hash_name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + + if (snprintf(inst->alg.base.cra_driver_name, CRYPTO_MAX_ALG_NAME, + "pkcs1(%s,%s)", rsa_alg->base.cra_driver_name, + hash_name) >= CRYPTO_MAX_ALG_NAME) + goto err_free_inst; + + inst->alg.base.cra_priority = rsa_alg->base.cra_priority; + inst->alg.base.cra_ctxsize = sizeof(struct rsassa_pkcs1_ctx); + + inst->alg.init = rsassa_pkcs1_init_tfm; + inst->alg.exit = rsassa_pkcs1_exit_tfm; + + inst->alg.sign = rsassa_pkcs1_sign; + inst->alg.verify = rsassa_pkcs1_verify; + inst->alg.key_size = rsassa_pkcs1_key_size; + inst->alg.set_pub_key = rsassa_pkcs1_set_pub_key; + inst->alg.set_priv_key = rsassa_pkcs1_set_priv_key; + + inst->free = rsassa_pkcs1_free; + + err = sig_register_instance(tmpl, inst); + if (err) { +err_free_inst: + rsassa_pkcs1_free(inst); + } + return err; +} + +struct crypto_template rsassa_pkcs1_tmpl = { + .name = "pkcs1", + .create = rsassa_pkcs1_create, + .module = THIS_MODULE, +}; + +MODULE_ALIAS_CRYPTO("pkcs1"); diff --git a/crypto/sig.c b/crypto/sig.c index 7645bedf3a1f..5e1f1f739da2 100644 --- a/crypto/sig.c +++ b/crypto/sig.c @@ -5,12 +5,10 @@ * Copyright (c) 2023 Herbert Xu <herbert@gondor.apana.org.au> */ -#include <crypto/akcipher.h> #include <crypto/internal/sig.h> #include <linux/cryptouser.h> #include <linux/kernel.h> #include <linux/module.h> -#include <linux/scatterlist.h> #include <linux/seq_file.h> #include <linux/string.h> #include <net/netlink.h> @@ -19,16 +17,35 @@ #define CRYPTO_ALG_TYPE_SIG_MASK 0x0000000e -static const struct crypto_type crypto_sig_type; +static void crypto_sig_exit_tfm(struct crypto_tfm *tfm) +{ + struct crypto_sig *sig = __crypto_sig_tfm(tfm); + struct sig_alg *alg = crypto_sig_alg(sig); + + alg->exit(sig); +} static int crypto_sig_init_tfm(struct crypto_tfm *tfm) { - if (tfm->__crt_alg->cra_type != &crypto_sig_type) - return crypto_init_akcipher_ops_sig(tfm); + struct crypto_sig *sig = __crypto_sig_tfm(tfm); + struct sig_alg *alg = crypto_sig_alg(sig); + + if (alg->exit) + sig->base.exit = crypto_sig_exit_tfm; + + if (alg->init) + return alg->init(sig); return 0; } +static void crypto_sig_free_instance(struct crypto_instance *inst) +{ + struct sig_instance *sig = sig_instance(inst); + + sig->free(sig); +} + static void __maybe_unused crypto_sig_show(struct seq_file *m, struct crypto_alg *alg) { @@ -38,16 +55,17 @@ static void __maybe_unused crypto_sig_show(struct seq_file *m, static int __maybe_unused crypto_sig_report(struct sk_buff *skb, struct crypto_alg *alg) { - struct crypto_report_akcipher rsig = {}; + struct crypto_report_sig rsig = {}; strscpy(rsig.type, "sig", sizeof(rsig.type)); - return nla_put(skb, CRYPTOCFGA_REPORT_AKCIPHER, sizeof(rsig), &rsig); + return nla_put(skb, CRYPTOCFGA_REPORT_SIG, sizeof(rsig), &rsig); } static const struct crypto_type crypto_sig_type = { .extsize = crypto_alg_extsize, .init_tfm = crypto_sig_init_tfm, + .free = crypto_sig_free_instance, #ifdef CONFIG_PROC_FS .show = crypto_sig_show, #endif @@ -66,74 +84,95 @@ struct crypto_sig *crypto_alloc_sig(const char *alg_name, u32 type, u32 mask) } EXPORT_SYMBOL_GPL(crypto_alloc_sig); -int crypto_sig_maxsize(struct crypto_sig *tfm) +static int sig_default_sign(struct crypto_sig *tfm, + const void *src, unsigned int slen, + void *dst, unsigned int dlen) +{ + return -ENOSYS; +} + +static int sig_default_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *dst, unsigned int dlen) { - struct crypto_akcipher **ctx = crypto_sig_ctx(tfm); + return -ENOSYS; +} - return crypto_akcipher_maxsize(*ctx); +static int sig_default_set_key(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + return -ENOSYS; } -EXPORT_SYMBOL_GPL(crypto_sig_maxsize); -int crypto_sig_sign(struct crypto_sig *tfm, - const void *src, unsigned int slen, - void *dst, unsigned int dlen) +static int sig_prepare_alg(struct sig_alg *alg) { - struct crypto_akcipher **ctx = crypto_sig_ctx(tfm); - struct crypto_akcipher_sync_data data = { - .tfm = *ctx, - .src = src, - .dst = dst, - .slen = slen, - .dlen = dlen, - }; - - return crypto_akcipher_sync_prep(&data) ?: - crypto_akcipher_sync_post(&data, - crypto_akcipher_sign(data.req)); + struct crypto_alg *base = &alg->base; + + if (!alg->sign) + alg->sign = sig_default_sign; + if (!alg->verify) + alg->verify = sig_default_verify; + if (!alg->set_priv_key) + alg->set_priv_key = sig_default_set_key; + if (!alg->set_pub_key) + return -EINVAL; + if (!alg->key_size) + return -EINVAL; + if (!alg->max_size) + alg->max_size = alg->key_size; + if (!alg->digest_size) + alg->digest_size = alg->key_size; + + base->cra_type = &crypto_sig_type; + base->cra_flags &= ~CRYPTO_ALG_TYPE_MASK; + base->cra_flags |= CRYPTO_ALG_TYPE_SIG; + + return 0; } -EXPORT_SYMBOL_GPL(crypto_sig_sign); -int crypto_sig_verify(struct crypto_sig *tfm, - const void *src, unsigned int slen, - const void *digest, unsigned int dlen) +int crypto_register_sig(struct sig_alg *alg) { - struct crypto_akcipher **ctx = crypto_sig_ctx(tfm); - struct crypto_akcipher_sync_data data = { - .tfm = *ctx, - .src = src, - .slen = slen, - .dlen = dlen, - }; + struct crypto_alg *base = &alg->base; int err; - err = crypto_akcipher_sync_prep(&data); + err = sig_prepare_alg(alg); if (err) return err; - memcpy(data.buf + slen, digest, dlen); + return crypto_register_alg(base); +} +EXPORT_SYMBOL_GPL(crypto_register_sig); - return crypto_akcipher_sync_post(&data, - crypto_akcipher_verify(data.req)); +void crypto_unregister_sig(struct sig_alg *alg) +{ + crypto_unregister_alg(&alg->base); } -EXPORT_SYMBOL_GPL(crypto_sig_verify); +EXPORT_SYMBOL_GPL(crypto_unregister_sig); -int crypto_sig_set_pubkey(struct crypto_sig *tfm, - const void *key, unsigned int keylen) +int sig_register_instance(struct crypto_template *tmpl, + struct sig_instance *inst) { - struct crypto_akcipher **ctx = crypto_sig_ctx(tfm); + int err; + + if (WARN_ON(!inst->free)) + return -EINVAL; + + err = sig_prepare_alg(&inst->alg); + if (err) + return err; - return crypto_akcipher_set_pub_key(*ctx, key, keylen); + return crypto_register_instance(tmpl, sig_crypto_instance(inst)); } -EXPORT_SYMBOL_GPL(crypto_sig_set_pubkey); +EXPORT_SYMBOL_GPL(sig_register_instance); -int crypto_sig_set_privkey(struct crypto_sig *tfm, - const void *key, unsigned int keylen) +int crypto_grab_sig(struct crypto_sig_spawn *spawn, + struct crypto_instance *inst, + const char *name, u32 type, u32 mask) { - struct crypto_akcipher **ctx = crypto_sig_ctx(tfm); - - return crypto_akcipher_set_priv_key(*ctx, key, keylen); + spawn->base.frontend = &crypto_sig_type; + return crypto_grab_spawn(&spawn->base, inst, name, type, mask); } -EXPORT_SYMBOL_GPL(crypto_sig_set_privkey); +EXPORT_SYMBOL_GPL(crypto_grab_sig); MODULE_LICENSE("GPL"); MODULE_DESCRIPTION("Public Key Signature Algorithms"); diff --git a/crypto/testmgr.c b/crypto/testmgr.c index 2f5f6b52b2d4..3699fc822778 100644 --- a/crypto/testmgr.c +++ b/crypto/testmgr.c @@ -33,6 +33,7 @@ #include <crypto/akcipher.h> #include <crypto/kpp.h> #include <crypto/acompress.h> +#include <crypto/sig.h> #include <crypto/internal/cipher.h> #include <crypto/internal/simd.h> @@ -131,6 +132,11 @@ struct akcipher_test_suite { unsigned int count; }; +struct sig_test_suite { + const struct sig_testvec *vecs; + unsigned int count; +}; + struct kpp_test_suite { const struct kpp_testvec *vecs; unsigned int count; @@ -151,6 +157,7 @@ struct alg_test_desc { struct cprng_test_suite cprng; struct drbg_test_suite drbg; struct akcipher_test_suite akcipher; + struct sig_test_suite sig; struct kpp_test_suite kpp; } suite; }; @@ -4123,11 +4130,9 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, struct crypto_wait wait; unsigned int out_len_max, out_len = 0; int err = -ENOMEM; - struct scatterlist src, dst, src_tab[3]; - const char *m, *c; - unsigned int m_size, c_size; - const char *op; - u8 *key, *ptr; + struct scatterlist src, dst, src_tab[2]; + const char *c; + unsigned int c_size; if (testmgr_alloc_buf(xbuf)) return err; @@ -4138,92 +4143,53 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, crypto_init_wait(&wait); - key = kmalloc(vecs->key_len + sizeof(u32) * 2 + vecs->param_len, - GFP_KERNEL); - if (!key) - goto free_req; - memcpy(key, vecs->key, vecs->key_len); - ptr = key + vecs->key_len; - ptr = test_pack_u32(ptr, vecs->algo); - ptr = test_pack_u32(ptr, vecs->param_len); - memcpy(ptr, vecs->params, vecs->param_len); - if (vecs->public_key_vec) - err = crypto_akcipher_set_pub_key(tfm, key, vecs->key_len); + err = crypto_akcipher_set_pub_key(tfm, vecs->key, + vecs->key_len); else - err = crypto_akcipher_set_priv_key(tfm, key, vecs->key_len); + err = crypto_akcipher_set_priv_key(tfm, vecs->key, + vecs->key_len); if (err) - goto free_key; + goto free_req; - /* - * First run test which do not require a private key, such as - * encrypt or verify. - */ + /* First run encrypt test which does not require a private key */ err = -ENOMEM; out_len_max = crypto_akcipher_maxsize(tfm); outbuf_enc = kzalloc(out_len_max, GFP_KERNEL); if (!outbuf_enc) - goto free_key; - - if (!vecs->siggen_sigver_test) { - m = vecs->m; - m_size = vecs->m_size; - c = vecs->c; - c_size = vecs->c_size; - op = "encrypt"; - } else { - /* Swap args so we could keep plaintext (digest) - * in vecs->m, and cooked signature in vecs->c. - */ - m = vecs->c; /* signature */ - m_size = vecs->c_size; - c = vecs->m; /* digest */ - c_size = vecs->m_size; - op = "verify"; - } + goto free_req; + + c = vecs->c; + c_size = vecs->c_size; err = -E2BIG; - if (WARN_ON(m_size > PAGE_SIZE)) + if (WARN_ON(vecs->m_size > PAGE_SIZE)) goto free_all; - memcpy(xbuf[0], m, m_size); + memcpy(xbuf[0], vecs->m, vecs->m_size); - sg_init_table(src_tab, 3); + sg_init_table(src_tab, 2); sg_set_buf(&src_tab[0], xbuf[0], 8); - sg_set_buf(&src_tab[1], xbuf[0] + 8, m_size - 8); - if (vecs->siggen_sigver_test) { - if (WARN_ON(c_size > PAGE_SIZE)) - goto free_all; - memcpy(xbuf[1], c, c_size); - sg_set_buf(&src_tab[2], xbuf[1], c_size); - akcipher_request_set_crypt(req, src_tab, NULL, m_size, c_size); - } else { - sg_init_one(&dst, outbuf_enc, out_len_max); - akcipher_request_set_crypt(req, src_tab, &dst, m_size, - out_len_max); - } + sg_set_buf(&src_tab[1], xbuf[0] + 8, vecs->m_size - 8); + sg_init_one(&dst, outbuf_enc, out_len_max); + akcipher_request_set_crypt(req, src_tab, &dst, vecs->m_size, + out_len_max); akcipher_request_set_callback(req, CRYPTO_TFM_REQ_MAY_BACKLOG, crypto_req_done, &wait); - err = crypto_wait_req(vecs->siggen_sigver_test ? - /* Run asymmetric signature verification */ - crypto_akcipher_verify(req) : - /* Run asymmetric encrypt */ - crypto_akcipher_encrypt(req), &wait); + err = crypto_wait_req(crypto_akcipher_encrypt(req), &wait); if (err) { - pr_err("alg: akcipher: %s test failed. err %d\n", op, err); + pr_err("alg: akcipher: encrypt test failed. err %d\n", err); goto free_all; } - if (!vecs->siggen_sigver_test && c) { + if (c) { if (req->dst_len != c_size) { - pr_err("alg: akcipher: %s test failed. Invalid output len\n", - op); + pr_err("alg: akcipher: encrypt test failed. Invalid output len\n"); err = -EINVAL; goto free_all; } /* verify that encrypted message is equal to expected */ if (memcmp(c, outbuf_enc, c_size) != 0) { - pr_err("alg: akcipher: %s test failed. Invalid output\n", - op); + pr_err("alg: akcipher: encrypt test failed. Invalid output\n"); hexdump(outbuf_enc, c_size); err = -EINVAL; goto free_all; @@ -4231,7 +4197,7 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, } /* - * Don't invoke (decrypt or sign) test which require a private key + * Don't invoke decrypt test which requires a private key * for vectors with only a public key. */ if (vecs->public_key_vec) { @@ -4244,13 +4210,12 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, goto free_all; } - if (!vecs->siggen_sigver_test && !c) { + if (!c) { c = outbuf_enc; c_size = req->dst_len; } err = -E2BIG; - op = vecs->siggen_sigver_test ? "sign" : "decrypt"; if (WARN_ON(c_size > PAGE_SIZE)) goto free_all; memcpy(xbuf[0], c, c_size); @@ -4260,34 +4225,29 @@ static int test_akcipher_one(struct crypto_akcipher *tfm, crypto_init_wait(&wait); akcipher_request_set_crypt(req, &src, &dst, c_size, out_len_max); - err = crypto_wait_req(vecs->siggen_sigver_test ? - /* Run asymmetric signature generation */ - crypto_akcipher_sign(req) : - /* Run asymmetric decrypt */ - crypto_akcipher_decrypt(req), &wait); + err = crypto_wait_req(crypto_akcipher_decrypt(req), &wait); if (err) { - pr_err("alg: akcipher: %s test failed. err %d\n", op, err); + pr_err("alg: akcipher: decrypt test failed. err %d\n", err); goto free_all; } out_len = req->dst_len; - if (out_len < m_size) { - pr_err("alg: akcipher: %s test failed. Invalid output len %u\n", - op, out_len); + if (out_len < vecs->m_size) { + pr_err("alg: akcipher: decrypt test failed. Invalid output len %u\n", + out_len); err = -EINVAL; goto free_all; } /* verify that decrypted message is equal to the original msg */ - if (memchr_inv(outbuf_dec, 0, out_len - m_size) || - memcmp(m, outbuf_dec + out_len - m_size, m_size)) { - pr_err("alg: akcipher: %s test failed. Invalid output\n", op); + if (memchr_inv(outbuf_dec, 0, out_len - vecs->m_size) || + memcmp(vecs->m, outbuf_dec + out_len - vecs->m_size, + vecs->m_size)) { + pr_err("alg: akcipher: decrypt test failed. Invalid output\n"); hexdump(outbuf_dec, out_len); err = -EINVAL; } free_all: kfree(outbuf_dec); kfree(outbuf_enc); -free_key: - kfree(key); free_req: akcipher_request_free(req); free_xbuf: @@ -4337,6 +4297,113 @@ static int alg_test_akcipher(const struct alg_test_desc *desc, return err; } +static int test_sig_one(struct crypto_sig *tfm, const struct sig_testvec *vecs) +{ + u8 *ptr, *key __free(kfree); + int err, sig_size; + + key = kmalloc(vecs->key_len + 2 * sizeof(u32) + vecs->param_len, + GFP_KERNEL); + if (!key) + return -ENOMEM; + + /* ecrdsa expects additional parameters appended to the key */ + memcpy(key, vecs->key, vecs->key_len); + ptr = key + vecs->key_len; + ptr = test_pack_u32(ptr, vecs->algo); + ptr = test_pack_u32(ptr, vecs->param_len); + memcpy(ptr, vecs->params, vecs->param_len); + + if (vecs->public_key_vec) + err = crypto_sig_set_pubkey(tfm, key, vecs->key_len); + else + err = crypto_sig_set_privkey(tfm, key, vecs->key_len); + if (err) + return err; + + /* + * Run asymmetric signature verification first + * (which does not require a private key) + */ + err = crypto_sig_verify(tfm, vecs->c, vecs->c_size, + vecs->m, vecs->m_size); + if (err) { + pr_err("alg: sig: verify test failed: err %d\n", err); + return err; + } + + /* + * Don't invoke sign test (which requires a private key) + * for vectors with only a public key. + */ + if (vecs->public_key_vec) + return 0; + + sig_size = crypto_sig_keysize(tfm); + if (sig_size < vecs->c_size) { + pr_err("alg: sig: invalid maxsize %u\n", sig_size); + return -EINVAL; + } + + u8 *sig __free(kfree) = kzalloc(sig_size, GFP_KERNEL); + if (!sig) + return -ENOMEM; + + /* Run asymmetric signature generation */ + err = crypto_sig_sign(tfm, vecs->m, vecs->m_size, sig, sig_size); + if (err) { + pr_err("alg: sig: sign test failed: err %d\n", err); + return err; + } + + /* Verify that generated signature equals cooked signature */ + if (memcmp(sig, vecs->c, vecs->c_size) || + memchr_inv(sig + vecs->c_size, 0, sig_size - vecs->c_size)) { + pr_err("alg: sig: sign test failed: invalid output\n"); + hexdump(sig, sig_size); + return -EINVAL; + } + + return 0; +} + +static int test_sig(struct crypto_sig *tfm, const char *alg, + const struct sig_testvec *vecs, unsigned int tcount) +{ + const char *algo = crypto_tfm_alg_driver_name(crypto_sig_tfm(tfm)); + int ret, i; + + for (i = 0; i < tcount; i++) { + ret = test_sig_one(tfm, vecs++); + if (ret) { + pr_err("alg: sig: test %d failed for %s: err %d\n", + i + 1, algo, ret); + return ret; + } + } + return 0; +} + +static int alg_test_sig(const struct alg_test_desc *desc, const char *driver, + u32 type, u32 mask) +{ + struct crypto_sig *tfm; + int err = 0; + + tfm = crypto_alloc_sig(driver, type, mask); + if (IS_ERR(tfm)) { + pr_err("alg: sig: Failed to load tfm for %s: %ld\n", + driver, PTR_ERR(tfm)); + return PTR_ERR(tfm); + } + if (desc->suite.sig.vecs) + err = test_sig(tfm, desc->alg, desc->suite.sig.vecs, + desc->suite.sig.count); + + crypto_free_sig(tfm); + return err; +} + static int alg_test_null(const struct alg_test_desc *desc, const char *driver, u32 type, u32 mask) { @@ -5126,36 +5193,36 @@ static const struct alg_test_desc alg_test_descs[] = { } }, { .alg = "ecdsa-nist-p192", - .test = alg_test_akcipher, + .test = alg_test_sig, .suite = { - .akcipher = __VECS(ecdsa_nist_p192_tv_template) + .sig = __VECS(ecdsa_nist_p192_tv_template) } }, { .alg = "ecdsa-nist-p256", - .test = alg_test_akcipher, + .test = alg_test_sig, .fips_allowed = 1, .suite = { - .akcipher = __VECS(ecdsa_nist_p256_tv_template) + .sig = __VECS(ecdsa_nist_p256_tv_template) } }, { .alg = "ecdsa-nist-p384", - .test = alg_test_akcipher, + .test = alg_test_sig, .fips_allowed = 1, .suite = { - .akcipher = __VECS(ecdsa_nist_p384_tv_template) + .sig = __VECS(ecdsa_nist_p384_tv_template) } }, { .alg = "ecdsa-nist-p521", - .test = alg_test_akcipher, + .test = alg_test_sig, .fips_allowed = 1, .suite = { - .akcipher = __VECS(ecdsa_nist_p521_tv_template) + .sig = __VECS(ecdsa_nist_p521_tv_template) } }, { .alg = "ecrdsa", - .test = alg_test_akcipher, + .test = alg_test_sig, .suite = { - .akcipher = __VECS(ecrdsa_tv_template) + .sig = __VECS(ecrdsa_tv_template) } }, { .alg = "essiv(authenc(hmac(sha256),cbc(aes)),sha256)", @@ -5448,40 +5515,68 @@ static const struct alg_test_desc alg_test_descs[] = { .hash = __VECS(nhpoly1305_tv_template) } }, { + .alg = "p1363(ecdsa-nist-p192)", + .test = alg_test_null, + }, { + .alg = "p1363(ecdsa-nist-p256)", + .test = alg_test_sig, + .fips_allowed = 1, + .suite = { + .sig = __VECS(p1363_ecdsa_nist_p256_tv_template) + } + }, { + .alg = "p1363(ecdsa-nist-p384)", + .test = alg_test_null, + .fips_allowed = 1, + }, { + .alg = "p1363(ecdsa-nist-p521)", + .test = alg_test_null, + .fips_allowed = 1, + }, { .alg = "pcbc(fcrypt)", .test = alg_test_skcipher, .suite = { .cipher = __VECS(fcrypt_pcbc_tv_template) } }, { - .alg = "pkcs1pad(rsa,sha224)", + .alg = "pkcs1(rsa,none)", + .test = alg_test_sig, + .suite = { + .sig = __VECS(pkcs1_rsa_none_tv_template) + } + }, { + .alg = "pkcs1(rsa,sha224)", .test = alg_test_null, .fips_allowed = 1, }, { - .alg = "pkcs1pad(rsa,sha256)", - .test = alg_test_akcipher, + .alg = "pkcs1(rsa,sha256)", + .test = alg_test_sig, .fips_allowed = 1, .suite = { - .akcipher = __VECS(pkcs1pad_rsa_tv_template) + .sig = __VECS(pkcs1_rsa_tv_template) } }, { - .alg = "pkcs1pad(rsa,sha3-256)", + .alg = "pkcs1(rsa,sha3-256)", + .test = alg_test_null, + .fips_allowed = 1, + }, { + .alg = "pkcs1(rsa,sha3-384)", .test = alg_test_null, .fips_allowed = 1, }, { - .alg = "pkcs1pad(rsa,sha3-384)", + .alg = "pkcs1(rsa,sha3-512)", .test = alg_test_null, .fips_allowed = 1, }, { - .alg = "pkcs1pad(rsa,sha3-512)", + .alg = "pkcs1(rsa,sha384)", .test = alg_test_null, .fips_allowed = 1, }, { - .alg = "pkcs1pad(rsa,sha384)", + .alg = "pkcs1(rsa,sha512)", .test = alg_test_null, .fips_allowed = 1, }, { - .alg = "pkcs1pad(rsa,sha512)", + .alg = "pkcs1pad(rsa)", .test = alg_test_null, .fips_allowed = 1, }, { @@ -5679,6 +5774,33 @@ static const struct alg_test_desc alg_test_descs[] = { .hash = __VECS(wp512_tv_template) } }, { + .alg = "x962(ecdsa-nist-p192)", + .test = alg_test_sig, + .suite = { + .sig = __VECS(x962_ecdsa_nist_p192_tv_template) + } + }, { + .alg = "x962(ecdsa-nist-p256)", + .test = alg_test_sig, + .fips_allowed = 1, + .suite = { + .sig = __VECS(x962_ecdsa_nist_p256_tv_template) + } + }, { + .alg = "x962(ecdsa-nist-p384)", + .test = alg_test_sig, + .fips_allowed = 1, + .suite = { + .sig = __VECS(x962_ecdsa_nist_p384_tv_template) + } + }, { + .alg = "x962(ecdsa-nist-p521)", + .test = alg_test_sig, + .fips_allowed = 1, + .suite = { + .sig = __VECS(x962_ecdsa_nist_p521_tv_template) + } + }, { .alg = "xcbc(aes)", .test = alg_test_hash, .suite = { diff --git a/crypto/testmgr.h b/crypto/testmgr.h index 9b38501a17b2..430d33d9ac13 100644 --- a/crypto/testmgr.h +++ b/crypto/testmgr.h @@ -21,6 +21,7 @@ #define _CRYPTO_TESTMGR_H #include <linux/oid_registry.h> +#include <crypto/internal/ecc.h> #define MAX_IVLEN 32 @@ -150,6 +151,16 @@ struct drbg_testvec { struct akcipher_testvec { const unsigned char *key; + const unsigned char *m; + const unsigned char *c; + unsigned int key_len; + unsigned int m_size; + unsigned int c_size; + bool public_key_vec; +}; + +struct sig_testvec { + const unsigned char *key; const unsigned char *params; const unsigned char *m; const unsigned char *c; @@ -158,7 +169,6 @@ struct akcipher_testvec { unsigned int m_size; unsigned int c_size; bool public_key_vec; - bool siggen_sigver_test; enum OID algo; }; @@ -647,26 +657,713 @@ static const struct akcipher_testvec rsa_tv_template[] = { } }; +#ifdef CONFIG_CPU_BIG_ENDIAN +#define be64_to_cpua(b1, b2, b3, b4, b5, b6, b7, b8) \ + 0x##b1, 0x##b2, 0x##b3, 0x##b4, 0x##b5, 0x##b6, 0x##b7, 0x##b8 +#else +#define be64_to_cpua(b1, b2, b3, b4, b5, b6, b7, b8) \ + 0x##b8, 0x##b7, 0x##b6, 0x##b5, 0x##b4, 0x##b3, 0x##b2, 0x##b1 +#endif + /* * ECDSA test vectors. */ -static const struct akcipher_testvec ecdsa_nist_p192_tv_template[] = { +static const struct sig_testvec ecdsa_nist_p192_tv_template[] = { { - .key = + .key = /* secp192r1(sha1) */ + "\x04\xf7\x46\xf8\x2f\x15\xf6\x22\x8e\xd7\x57\x4f\xcc\xe7\xbb\xc1" + "\xd4\x09\x73\xcf\xea\xd0\x15\x07\x3d\xa5\x8a\x8a\x95\x43\xe4\x68" + "\xea\xc6\x25\xc1\xc1\x01\x25\x4c\x7e\xc3\x3c\xa6\x04\x0a\xe7\x08" + "\x98", + .key_len = 49, + .m = + "\xcd\xb9\xd2\x1c\xb7\x6f\xcd\x44\xb3\xfd\x63\xea\xa3\x66\x7f\xae" + "\x63\x85\xe7\x82", + .m_size = 20, + .c = (const unsigned char[]){ + be64_to_cpua(ad, 59, ad, 88, 27, d6, 92, 6b), + be64_to_cpua(a0, 27, 91, c6, f6, 7f, c3, 09), + be64_to_cpua(ba, e5, 93, 83, 6e, b6, 3b, 63), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(86, 80, 6f, a5, 79, 77, da, d0), + be64_to_cpua(ef, 95, 52, 7b, a0, 0f, e4, 18), + be64_to_cpua(10, 68, 01, 9d, ba, ce, 83, 08), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp192r1(sha224) */ + "\x04\xb6\x4b\xb1\xd1\xac\xba\x24\x8f\x65\xb2\x60\x00\x90\xbf\xbd" + "\x78\x05\x73\xe9\x79\x1d\x6f\x7c\x0b\xd2\xc3\x93\xa7\x28\xe1\x75" + "\xf7\xd5\x95\x1d\x28\x10\xc0\x75\x50\x5c\x1a\x4f\x3f\x8f\xa5\xee" + "\xa3", + .key_len = 49, + .m = + "\x8d\xd6\xb8\x3e\xe5\xff\x23\xf6\x25\xa2\x43\x42\x74\x45\xa7\x40" + "\x3a\xff\x2f\xe1\xd3\xf6\x9f\xe8\x33\xcb\x12\x11", + .m_size = 28, + .c = (const unsigned char[]){ + be64_to_cpua(83, 7b, 12, e6, b6, 5b, cb, d4), + be64_to_cpua(14, f8, 11, 2b, 55, dc, ae, 37), + be64_to_cpua(5a, 8b, 82, 69, 7e, 8a, 0a, 09), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(a3, e3, 5c, 99, db, 92, 5b, 36), + be64_to_cpua(eb, c3, 92, 0f, 1e, 72, ee, c4), + be64_to_cpua(6a, 14, 4f, 53, 75, c8, 02, 48), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp192r1(sha256) */ + "\x04\xe2\x51\x24\x9b\xf7\xb6\x32\x82\x39\x66\x3d\x5b\xec\x3b\xae" + "\x0c\xd5\xf2\x67\xd1\xc7\xe1\x02\xe4\xbf\x90\x62\xb8\x55\x75\x56" + "\x69\x20\x5e\xcb\x4e\xca\x33\xd6\xcb\x62\x6b\x94\xa9\xa2\xe9\x58" + "\x91", + .key_len = 49, + .m = + "\x35\xec\xa1\xa0\x9e\x14\xde\x33\x03\xb6\xf6\xbd\x0c\x2f\xb2\xfd" + "\x1f\x27\x82\xa5\xd7\x70\x3f\xef\xa0\x82\x69\x8e\x73\x31\x8e\xd7", + .m_size = 32, + .c = (const unsigned char[]){ + be64_to_cpua(01, 48, fb, 5f, 72, 2a, d4, 8f), + be64_to_cpua(6b, 1a, 58, 56, f1, 8f, f7, fd), + be64_to_cpua(3f, 72, 3f, 1f, 42, d2, 3f, 1d), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(7d, 3a, 97, d9, cd, 1a, 6a, 49), + be64_to_cpua(32, dd, 41, 74, 6a, 51, c7, d9), + be64_to_cpua(b3, 69, 43, fd, 48, 19, 86, cf), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp192r1(sha384) */ + "\x04\x5a\x13\xfe\x68\x86\x4d\xf4\x17\xc7\xa4\xe5\x8c\x65\x57\xb7" + "\x03\x73\x26\x57\xfb\xe5\x58\x40\xd8\xfd\x49\x05\xab\xf1\x66\x1f" + "\xe2\x9d\x93\x9e\xc2\x22\x5a\x8b\x4f\xf3\x77\x22\x59\x7e\xa6\x4e" + "\x8b", + .key_len = 49, + .m = + "\x9d\x2e\x1a\x8f\xed\x6c\x4b\x61\xae\xac\xd5\x19\x79\xce\x67\xf9" + "\xa0\x34\xeb\xb0\x81\xf9\xd9\xdc\x6e\xb3\x5c\xa8\x69\xfc\x8a\x61" + "\x39\x81\xfb\xfd\x5c\x30\x6b\xa8\xee\xed\x89\xaf\xa3\x05\xe4\x78", + .m_size = 48, + .c = (const unsigned char[]){ + be64_to_cpua(dd, 15, bb, d6, 8c, a7, 03, 78), + be64_to_cpua(cf, 7f, 34, b4, b4, e5, c5, 00), + be64_to_cpua(f0, a3, 38, ce, 2b, f8, 9d, 1a), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(93, 12, 3b, 3b, 28, fb, 6d, e1), + be64_to_cpua(d1, 01, 77, 44, 5d, 53, a4, 7c), + be64_to_cpua(64, bc, 5a, 1f, 82, 96, 61, d7), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp192r1(sha512) */ + "\x04\xd5\xf2\x6e\xc3\x94\x5c\x52\xbc\xdf\x86\x6c\x14\xd1\xca\xea" + "\xcc\x72\x3a\x8a\xf6\x7a\x3a\x56\x36\x3b\xca\xc6\x94\x0e\x17\x1d" + "\x9e\xa0\x58\x28\xf9\x4b\xe6\xd1\xa5\x44\x91\x35\x0d\xe7\xf5\x11" + "\x57", + .key_len = 49, + .m = + "\xd5\x4b\xe9\x36\xda\xd8\x6e\xc0\x50\x03\xbe\x00\x43\xff\xf0\x23" + "\xac\xa2\x42\xe7\x37\x77\x79\x52\x8f\x3e\xc0\x16\xc1\xfc\x8c\x67" + "\x16\xbc\x8a\x5d\x3b\xd3\x13\xbb\xb6\xc0\x26\x1b\xeb\x33\xcc\x70" + "\x4a\xf2\x11\x37\xe8\x1b\xba\x55\xac\x69\xe1\x74\x62\x7c\x6e\xb5", + .m_size = 64, + .c = (const unsigned char[]){ + be64_to_cpua(2b, 11, 2d, 1c, b6, 06, c9, 6c), + be64_to_cpua(dd, 3f, 07, 87, 12, a0, d4, ac), + be64_to_cpua(88, 5b, 8f, 59, 43, bf, cf, c6), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(28, 6a, df, 97, fd, 82, 76, 24), + be64_to_cpua(a9, 14, 2a, 5e, f5, e5, fb, 72), + be64_to_cpua(73, b4, 22, 9a, 98, 73, 3c, 83), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, +}; + +static const struct sig_testvec ecdsa_nist_p256_tv_template[] = { + { + .key = /* secp256r1(sha1) */ + "\x04\xb9\x7b\xbb\xd7\x17\x64\xd2\x7e\xfc\x81\x5d\x87\x06\x83\x41" + "\x22\xd6\x9a\xaa\x87\x17\xec\x4f\x63\x55\x2f\x94\xba\xdd\x83\xe9" + "\x34\x4b\xf3\xe9\x91\x13\x50\xb6\xcb\xca\x62\x08\xe7\x3b\x09\xdc" + "\xc3\x63\x4b\x2d\xb9\x73\x53\xe4\x45\xe6\x7c\xad\xe7\x6b\xb0\xe8" + "\xaf", + .key_len = 65, + .m = + "\xc2\x2b\x5f\x91\x78\x34\x26\x09\x42\x8d\x6f\x51\xb2\xc5\xaf\x4c" + "\x0b\xde\x6a\x42", + .m_size = 20, + .c = (const unsigned char[]){ + be64_to_cpua(ee, ca, 6a, 52, 0e, 48, 4d, cc), + be64_to_cpua(f7, d4, ad, 8d, 94, 5a, 69, 89), + be64_to_cpua(cf, d4, e7, b7, f0, 82, 56, 41), + be64_to_cpua(f9, 25, ce, 9f, 3a, a6, 35, 81), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(fb, 9d, 8b, de, d4, 8d, 6f, ad), + be64_to_cpua(f1, 03, 03, f3, 3b, e2, 73, f7), + be64_to_cpua(8a, fa, 54, 93, 29, a7, 70, 86), + be64_to_cpua(d7, e4, ef, 52, 66, d3, 5b, 9d), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp256r1(sha224) */ + "\x04\x8b\x6d\xc0\x33\x8e\x2d\x8b\x67\xf5\xeb\xc4\x7f\xa0\xf5\xd9" + "\x7b\x03\xa5\x78\x9a\xb5\xea\x14\xe4\x23\xd0\xaf\xd7\x0e\x2e\xa0" + "\xc9\x8b\xdb\x95\xf8\xb3\xaf\xac\x00\x2c\x2c\x1f\x7a\xfd\x95\x88" + "\x43\x13\xbf\xf3\x1c\x05\x1a\x14\x18\x09\x3f\xd6\x28\x3e\xc5\xa0" + "\xd4", + .key_len = 65, + .m = + "\x1a\x15\xbc\xa3\xe4\xed\x3a\xb8\x23\x67\xc6\xc4\x34\xf8\x6c\x41" + "\x04\x0b\xda\xc5\x77\xfa\x1c\x2d\xe6\x2c\x3b\xe0", + .m_size = 28, + .c = (const unsigned char[]){ + be64_to_cpua(7d, 25, d8, 25, f5, 81, d2, 1e), + be64_to_cpua(34, 62, 79, cb, 6a, 91, 67, 2e), + be64_to_cpua(ae, ce, 77, 59, 1a, db, 59, d5), + be64_to_cpua(20, 43, fa, c0, 9f, 9d, 7b, e7), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(ce, d5, 2e, 8b, de, 5a, 04, 0e), + be64_to_cpua(bf, 50, 05, 58, 39, 0e, 26, 92), + be64_to_cpua(76, 20, 4a, 77, 22, ec, c8, 66), + be64_to_cpua(5f, f8, 74, f8, 57, d0, 5e, 54), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp256r1(sha256) */ + "\x04\xf1\xea\xc4\x53\xf3\xb9\x0e\x9f\x7e\xad\xe3\xea\xd7\x0e\x0f" + "\xd6\x98\x9a\xca\x92\x4d\x0a\x80\xdb\x2d\x45\xc7\xec\x4b\x97\x00" + "\x2f\xe9\x42\x6c\x29\xdc\x55\x0e\x0b\x53\x12\x9b\x2b\xad\x2c\xe9" + "\x80\xe6\xc5\x43\xc2\x1d\x5e\xbb\x65\x21\x50\xb6\x37\xb0\x03\x8e" + "\xb8", + .key_len = 65, + .m = + "\x8f\x43\x43\x46\x64\x8f\x6b\x96\xdf\x89\xdd\xa9\x01\xc5\x17\x6b" + "\x10\xa6\xd8\x39\x61\xdd\x3c\x1a\xc8\x8b\x59\xb2\xdc\x32\x7a\xa4", + .m_size = 32, + .c = (const unsigned char[]){ + be64_to_cpua(91, dc, 02, 67, dc, 0c, d0, 82), + be64_to_cpua(ac, 44, c3, e8, 24, 11, 2d, a4), + be64_to_cpua(09, dc, 29, 63, a8, 1a, ad, fc), + be64_to_cpua(08, 31, fa, 74, 0d, 1d, 21, 5d), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(4f, 2a, 65, 35, 23, e3, 1d, fa), + be64_to_cpua(0a, 6e, 1b, c4, af, e1, 83, c3), + be64_to_cpua(f9, a9, 81, ac, 4a, 50, d0, 91), + be64_to_cpua(bd, ff, ce, ee, 42, c3, 97, ff), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp256r1(sha384) */ + "\x04\xc5\xc6\xea\x60\xc9\xce\xad\x02\x8d\xf5\x3e\x24\xe3\x52\x1d" + "\x28\x47\x3b\xc3\x6b\xa4\x99\x35\x99\x11\x88\x88\xc8\xf4\xee\x7e" + "\x8c\x33\x8f\x41\x03\x24\x46\x2b\x1a\x82\xf9\x9f\xe1\x97\x1b\x00" + "\xda\x3b\x24\x41\xf7\x66\x33\x58\x3d\x3a\x81\xad\xcf\x16\xe9\xe2" + "\x7c", + .key_len = 65, + .m = + "\x3e\x78\x70\xfb\xcd\x66\xba\x91\xa1\x79\xff\x1e\x1c\x6b\x78\xe6" + "\xc0\x81\x3a\x65\x97\x14\x84\x36\x14\x1a\x9a\xb7\xc5\xab\x84\x94" + "\x5e\xbb\x1b\x34\x71\xcb\x41\xe1\xf6\xfc\x92\x7b\x34\xbb\x86\xbb", + .m_size = 48, + .c = (const unsigned char[]){ + be64_to_cpua(f2, e4, 6c, c7, 94, b1, d5, fe), + be64_to_cpua(08, b2, 6b, 24, 94, 48, 46, 5e), + be64_to_cpua(d0, 2e, 95, 54, d1, 95, 64, 93), + be64_to_cpua(8e, f3, 6f, dc, f8, 69, a6, 2e), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(c0, 60, 11, 92, dc, 17, 89, 12), + be64_to_cpua(69, f4, 3b, 4f, 47, cf, 9b, 16), + be64_to_cpua(19, fb, 5f, 92, f4, c9, 23, 37), + be64_to_cpua(eb, a7, 80, 26, dc, f9, 3a, 44), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp256r1(sha512) */ + "\x04\xd7\x27\x46\x49\xf6\x26\x85\x12\x40\x76\x8e\xe2\xe6\x2a\x7a" + "\x83\xb1\x4e\x7a\xeb\x3b\x5c\x67\x4a\xb5\xa4\x92\x8c\x69\xff\x38" + "\xee\xd9\x4e\x13\x29\x59\xad\xde\x6b\xbb\x45\x31\xee\xfd\xd1\x1b" + "\x64\xd3\xb5\xfc\xaf\x9b\x4b\x88\x3b\x0e\xb7\xd6\xdf\xf1\xd5\x92" + "\xbf", + .key_len = 65, + .m = + "\x57\xb7\x9e\xe9\x05\x0a\x8c\x1b\xc9\x13\xe5\x4a\x24\xc7\xe2\xe9" + "\x43\xc3\xd1\x76\x62\xf4\x98\x1a\x9c\x13\xb0\x20\x1b\xe5\x39\xca" + "\x4f\xd9\x85\x34\x95\xa2\x31\xbc\xbb\xde\xdd\x76\xbb\x61\xe3\xcf" + "\x9d\xc0\x49\x7a\xf3\x7a\xc4\x7d\xa8\x04\x4b\x8d\xb4\x4d\x5b\xd6", + .m_size = 64, + .c = (const unsigned char[]){ + be64_to_cpua(76, f6, 04, 99, 09, 37, 4d, fa), + be64_to_cpua(ed, 8c, 73, 30, 6c, 22, b3, 97), + be64_to_cpua(40, ea, 44, 81, 00, 4e, 29, 08), + be64_to_cpua(b8, 6d, 87, 81, 43, df, fb, 9f), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(76, 31, 79, 4a, e9, 81, 6a, ee), + be64_to_cpua(5c, ad, c3, 78, 1c, c2, c1, 19), + be64_to_cpua(f8, 00, dd, ab, d4, c0, 2b, e6), + be64_to_cpua(1e, b9, 75, 31, f6, 04, a5, 4d), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, +}; + +static const struct sig_testvec ecdsa_nist_p384_tv_template[] = { + { + .key = /* secp384r1(sha1) */ + "\x04\x89\x25\xf3\x97\x88\xcb\xb0\x78\xc5\x72\x9a\x14\x6e\x7a\xb1" + "\x5a\xa5\x24\xf1\x95\x06\x9e\x28\xfb\xc4\xb9\xbe\x5a\x0d\xd9\x9f" + "\xf3\xd1\x4d\x2d\x07\x99\xbd\xda\xa7\x66\xec\xbb\xea\xba\x79\x42" + "\xc9\x34\x89\x6a\xe7\x0b\xc3\xf2\xfe\x32\x30\xbe\xba\xf9\xdf\x7e" + "\x4b\x6a\x07\x8e\x26\x66\x3f\x1d\xec\xa2\x57\x91\x51\xdd\x17\x0e" + "\x0b\x25\xd6\x80\x5c\x3b\xe6\x1a\x98\x48\x91\x45\x7a\x73\xb0\xc3" + "\xf1", + .key_len = 97, + .m = + "\x12\x55\x28\xf0\x77\xd5\xb6\x21\x71\x32\x48\xcd\x28\xa8\x25\x22" + "\x3a\x69\xc1\x93", + .m_size = 20, + .c = (const unsigned char[]){ + be64_to_cpua(ec, 7c, 7e, d0, 87, d7, d7, 6e), + be64_to_cpua(78, f1, 4c, 26, e6, 5b, 86, cf), + be64_to_cpua(3a, c6, f1, 32, 3c, ce, 70, 2b), + be64_to_cpua(8d, 26, 8e, ae, 63, 3f, bc, 20), + be64_to_cpua(57, 55, 07, 20, 43, 30, de, a0), + be64_to_cpua(f5, 0f, 24, 4c, 07, 93, 6f, 21), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(79, 12, 2a, b7, c5, 15, 92, c5), + be64_to_cpua(4a, a1, 59, f1, 1c, a4, 58, 26), + be64_to_cpua(74, a0, 0f, bf, af, c3, 36, 76), + be64_to_cpua(df, 28, 8c, 1b, fa, f9, 95, 88), + be64_to_cpua(5f, 63, b1, be, 5e, 4c, 0e, a1), + be64_to_cpua(cd, bb, 7e, 81, 5d, 8f, 63, c0), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp384r1(sha224) */ + "\x04\x69\x6c\xcf\x62\xee\xd0\x0d\xe5\xb5\x2f\x70\x54\xcf\x26\xa0" + "\xd9\x98\x8d\x92\x2a\xab\x9b\x11\xcb\x48\x18\xa1\xa9\x0d\xd5\x18" + "\x3e\xe8\x29\x6e\xf6\xe4\xb5\x8e\xc7\x4a\xc2\x5f\x37\x13\x99\x05" + "\xb6\xa4\x9d\xf9\xfb\x79\x41\xe7\xd7\x96\x9f\x73\x3b\x39\x43\xdc" + "\xda\xf4\x06\xb9\xa5\x29\x01\x9d\x3b\xe1\xd8\x68\x77\x2a\xf4\x50" + "\x6b\x93\x99\x6c\x66\x4c\x42\x3f\x65\x60\x6c\x1c\x0b\x93\x9b\x9d" + "\xe0", + .key_len = 97, + .m = + "\x12\x80\xb6\xeb\x25\xe2\x3d\xf0\x21\x32\x96\x17\x3a\x38\x39\xfd" + "\x1f\x05\x34\x7b\xb8\xf9\x71\x66\x03\x4f\xd5\xe5", + .m_size = 28, + .c = (const unsigned char[]){ + be64_to_cpua(3f, dd, 15, 1b, 68, 2b, 9d, 8b), + be64_to_cpua(c9, 9c, 11, b8, 10, 01, c5, 41), + be64_to_cpua(c5, da, b4, e3, 93, 07, e0, 99), + be64_to_cpua(97, f1, c8, 72, 26, cf, 5a, 5e), + be64_to_cpua(ec, cb, e4, 89, 47, b2, f7, bc), + be64_to_cpua(8a, 51, 84, ce, 13, 1e, d2, dc), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(88, 2b, 82, 26, 5e, 1c, da, fb), + be64_to_cpua(9f, 19, d0, 42, 8b, 93, c2, 11), + be64_to_cpua(4d, d0, c6, 6e, b0, e9, fc, 14), + be64_to_cpua(df, d8, 68, a2, 64, 42, 65, f3), + be64_to_cpua(4b, 00, 08, 31, 6c, f5, d5, f6), + be64_to_cpua(8b, 03, 2c, fc, 1f, d1, a9, a4), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp384r1(sha256) */ + "\x04\xee\xd6\xda\x3e\x94\x90\x00\x27\xed\xf8\x64\x55\xd6\x51\x9a" + "\x1f\x52\x00\x63\x78\xf1\xa9\xfd\x75\x4c\x9e\xb2\x20\x1a\x91\x5a" + "\xba\x7a\xa3\xe5\x6c\xb6\x25\x68\x4b\xe8\x13\xa6\x54\x87\x2c\x0e" + "\xd0\x83\x95\xbc\xbf\xc5\x28\x4f\x77\x1c\x46\xa6\xf0\xbc\xd4\xa4" + "\x8d\xc2\x8f\xb3\x32\x37\x40\xd6\xca\xf8\xae\x07\x34\x52\x39\x52" + "\x17\xc3\x34\x29\xd6\x40\xea\x5c\xb9\x3f\xfb\x32\x2e\x12\x33\xbc" + "\xab", + .key_len = 97, + .m = + "\xaa\xe7\xfd\x03\x26\xcb\x94\x71\xe4\xce\x0f\xc5\xff\xa6\x29\xa3" + "\xe1\xcc\x4c\x35\x4e\xde\xca\x80\xab\x26\x0c\x25\xe6\x68\x11\xc2", + .m_size = 32, + .c = (const unsigned char[]){ + be64_to_cpua(c8, 8d, 2c, 79, 3a, 8e, 32, c4), + be64_to_cpua(b6, c6, fc, 70, 2e, 66, 3c, 77), + be64_to_cpua(af, 06, 3f, 84, 04, e2, f9, 67), + be64_to_cpua(cc, 47, 53, 87, bc, bd, 83, 3f), + be64_to_cpua(8e, 3f, 7e, ce, 0a, 9b, aa, 59), + be64_to_cpua(08, 09, 12, 9d, 6e, 96, 64, a6), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(10, 0e, f4, 1f, 39, ca, 4d, 43), + be64_to_cpua(4f, 8d, de, 1e, 93, 8d, 95, bb), + be64_to_cpua(15, 68, c0, 75, 3e, 23, 5e, 36), + be64_to_cpua(dd, ce, bc, b2, 97, f4, 9c, f3), + be64_to_cpua(26, a2, b0, 89, 42, 0a, da, d9), + be64_to_cpua(40, 34, b8, 90, a9, 80, ab, 47), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp384r1(sha384) */ + "\x04\x3a\x2f\x62\xe7\x1a\xcf\x24\xd0\x0b\x7c\xe0\xed\x46\x0a\x4f" + "\x74\x16\x43\xe9\x1a\x25\x7c\x55\xff\xf0\x29\x68\x66\x20\x91\xf9" + "\xdb\x2b\xf6\xb3\x6c\x54\x01\xca\xc7\x6a\x5c\x0d\xeb\x68\xd9\x3c" + "\xf1\x01\x74\x1f\xf9\x6c\xe5\x5b\x60\xe9\x7f\x5d\xb3\x12\x80\x2a" + "\xd8\x67\x92\xc9\x0e\x4c\x4c\x6b\xa1\xb2\xa8\x1e\xac\x1c\x97\xd9" + "\x21\x67\xe5\x1b\x5a\x52\x31\x68\xd6\xee\xf0\x19\xb0\x55\xed\x89" + "\x9e", + .key_len = 97, + .m = + "\x8d\xf2\xc0\xe9\xa8\xf3\x8e\x44\xc4\x8c\x1a\xa0\xb8\xd7\x17\xdf" + "\xf2\x37\x1b\xc6\xe3\xf5\x62\xcc\x68\xf5\xd5\x0b\xbf\x73\x2b\xb1" + "\xb0\x4c\x04\x00\x31\xab\xfe\xc8\xd6\x09\xc8\xf2\xea\xd3\x28\xff", + .m_size = 48, + .c = (const unsigned char[]){ + be64_to_cpua(a2, a4, c8, f2, ea, 9d, 11, 1f), + be64_to_cpua(3b, 1f, 07, 8f, 15, 02, fe, 1d), + be64_to_cpua(29, e6, fb, ca, 8c, d6, b6, b4), + be64_to_cpua(2d, 7a, 91, 5f, 49, 2d, 22, 08), + be64_to_cpua(ee, 2e, 62, 35, 46, fa, 00, d8), + be64_to_cpua(9b, 28, 68, c0, a1, ea, 8c, 50), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(ab, 8d, 4e, de, e6, 6d, 9b, 66), + be64_to_cpua(96, 17, 04, c9, 05, 77, f1, 8e), + be64_to_cpua(44, 92, 8c, 86, 99, 65, b3, 97), + be64_to_cpua(71, cd, 8f, 18, 99, f0, 0f, 13), + be64_to_cpua(bf, e3, 75, 24, 49, ac, fb, c8), + be64_to_cpua(fc, 50, f6, 43, bd, 50, 82, 0e), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, { + .key = /* secp384r1(sha512) */ + "\x04\xb4\xe7\xc1\xeb\x64\x25\x22\x46\xc3\x86\x61\x80\xbe\x1e\x46" + "\xcb\xf6\x05\xc2\xee\x73\x83\xbc\xea\x30\x61\x4d\x40\x05\x41\xf4" + "\x8c\xe3\x0e\x5c\xf0\x50\xf2\x07\x19\xe8\x4f\x25\xbe\xee\x0c\x95" + "\x54\x36\x86\xec\xc2\x20\x75\xf3\x89\xb5\x11\xa1\xb7\xf5\xaf\xbe" + "\x81\xe4\xc3\x39\x06\xbd\xe4\xfe\x68\x1c\x6d\x99\x2b\x1b\x63\xfa" + "\xdf\x42\x5c\xc2\x5a\xc7\x0c\xf4\x15\xf7\x1b\xa3\x2e\xd7\x00\xac" + "\xa3", + .key_len = 97, + .m = + "\xe8\xb7\x52\x7d\x1a\x44\x20\x05\x53\x6b\x3a\x68\xf2\xe7\x6c\xa1" + "\xae\x9d\x84\xbb\xba\x52\x43\x3e\x2c\x42\x78\x49\xbf\x78\xb2\x71" + "\xeb\xe1\xe0\xe8\x42\x7b\x11\xad\x2b\x99\x05\x1d\x36\xe6\xac\xfc" + "\x55\x73\xf0\x15\x63\x39\xb8\x6a\x6a\xc5\x91\x5b\xca\x6a\xa8\x0e", + .m_size = 64, + .c = (const unsigned char[]){ + be64_to_cpua(3e, b3, c7, a8, b3, 17, 77, d1), + be64_to_cpua(dc, 2b, 43, 0e, 6a, b3, 53, 6f), + be64_to_cpua(4c, fc, 6f, 80, e3, af, b3, d9), + be64_to_cpua(9a, 02, de, 93, e8, 83, e4, 84), + be64_to_cpua(4d, c6, ef, da, 02, e7, 0f, 52), + be64_to_cpua(00, 1d, 20, 94, 77, fe, 31, fa), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(4e, 45, cf, 3c, 93, ff, 50, 5d), + be64_to_cpua(34, e4, 8b, 80, a5, b6, da, 2c), + be64_to_cpua(c4, 6a, 03, 5f, 8d, 7a, f9, fb), + be64_to_cpua(ec, 63, e3, 0c, ec, 50, dc, cc), + be64_to_cpua(de, 3a, 3d, 16, af, b4, 52, 6a), + be64_to_cpua(63, f6, f0, 3d, 5f, 5f, 99, 3f), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 00) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, +}; + +static const struct sig_testvec ecdsa_nist_p521_tv_template[] = { + { + .key = /* secp521r1(sha224) */ + "\x04\x01\x4f\x43\x18\xb6\xa9\xc9\x5d\x68\xd3\xa9\x42\xf8\x98\xc0" + "\xd2\xd1\xa9\x50\x3b\xe8\xc4\x40\xe6\x11\x78\x88\x4b\xbd\x76\xa7" + "\x9a\xe0\xdd\x31\xa4\x67\x78\x45\x33\x9e\x8c\xd1\xc7\x44\xac\x61" + "\x68\xc8\x04\xe7\x5c\x79\xb1\xf1\x41\x0c\x71\xc0\x53\xa8\xbc\xfb" + "\xf5\xca\xd4\x01\x40\xfd\xa3\x45\xda\x08\xe0\xb4\xcb\x28\x3b\x0a" + "\x02\x35\x5f\x02\x9f\x3f\xcd\xef\x08\x22\x40\x97\x74\x65\xb7\x76" + "\x85\xc7\xc0\x5c\xfb\x81\xe1\xa5\xde\x0c\x4e\x8b\x12\x31\xb6\x47" + "\xed\x37\x0f\x99\x3f\x26\xba\xa3\x8e\xff\x79\x34\x7c\x3a\xfe\x1f" + "\x3b\x83\x82\x2f\x14", + .key_len = 133, + .m = + "\xa2\x3a\x6a\x8c\x7b\x3c\xf2\x51\xf8\xbe\x5f\x4f\x3b\x15\x05\xc4" + "\xb5\xbc\x19\xe7\x21\x85\xe9\x23\x06\x33\x62\xfb", + .m_size = 28, + .c = (const unsigned char[]){ + be64_to_cpua(46, 6b, c7, af, 7a, b9, 19, 0a), + be64_to_cpua(6c, a6, 9b, 89, 8b, 1e, fd, 09), + be64_to_cpua(98, 85, 29, 88, ff, 0b, 94, 94), + be64_to_cpua(18, c6, 37, 8a, cb, a7, d8, 7d), + be64_to_cpua(f8, 3f, 59, 0f, 74, f0, 3f, d8), + be64_to_cpua(e2, ef, 07, 92, ee, 60, 94, 06), + be64_to_cpua(35, f6, dc, 6d, 02, 7b, 22, ac), + be64_to_cpua(d6, 43, e7, ff, 42, b2, ba, 74), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 01), + be64_to_cpua(50, b1, a5, 98, 92, 2a, a5, 52), + be64_to_cpua(1c, ad, 22, da, 82, 00, 35, a3), + be64_to_cpua(0e, 64, cc, c4, e8, 43, d9, 0e), + be64_to_cpua(30, 90, 0f, 1c, 8f, 78, d3, 9f), + be64_to_cpua(26, 0b, 5f, 49, 32, 6b, 91, 99), + be64_to_cpua(0f, f8, 65, 97, 6b, 09, 4d, 22), + be64_to_cpua(5e, f9, 88, f3, d2, 32, 90, 57), + be64_to_cpua(26, 0d, 55, cd, 23, 1e, 7d, a0), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 3a) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, + { + .key = /* secp521r1(sha256) */ + "\x04\x01\x05\x3a\x6b\x3b\x5a\x0f\xa7\xb9\xb7\x32\x53\x4e\xe2\xae" + "\x0a\x52\xc5\xda\xdd\x5a\x79\x1c\x30\x2d\x33\x07\x79\xd5\x70\x14" + "\x61\x0c\xec\x26\x4d\xd8\x35\x57\x04\x1d\x88\x33\x4d\xce\x05\x36" + "\xa5\xaf\x56\x84\xfa\x0b\x9e\xff\x7b\x30\x4b\x92\x1d\x06\xf8\x81" + "\x24\x1e\x51\x00\x09\x21\x51\xf7\x46\x0a\x77\xdb\xb5\x0c\xe7\x9c" + "\xff\x27\x3c\x02\x71\xd7\x85\x36\xf1\xaa\x11\x59\xd8\xb8\xdc\x09" + "\xdc\x6d\x5a\x6f\x63\x07\x6c\xe1\xe5\x4d\x6e\x0f\x6e\xfb\x7c\x05" + "\x8a\xe9\x53\xa8\xcf\xce\x43\x0e\x82\x20\x86\xbc\x88\x9c\xb7\xe3" + "\xe6\x77\x1e\x1f\x8a", + .key_len = 133, + .m = + "\xcc\x97\x73\x0c\x73\xa2\x53\x2b\xfa\xd7\x83\x1d\x0c\x72\x1b\x39" + "\x80\x71\x8d\xdd\xc5\x9b\xff\x55\x32\x98\x25\xa2\x58\x2e\xb7\x73", + .m_size = 32, + .c = (const unsigned char[]){ + be64_to_cpua(de, 7e, d7, 59, 10, e9, d9, d5), + be64_to_cpua(38, 1f, 46, 0b, 04, 64, 34, 79), + be64_to_cpua(ae, ce, 54, 76, 9a, c2, 8f, b8), + be64_to_cpua(95, 35, 6f, 02, 0e, af, e1, 4c), + be64_to_cpua(56, 3c, f6, f0, d8, e1, b7, 5d), + be64_to_cpua(50, 9f, 7d, 1f, ca, 8b, a8, 2d), + be64_to_cpua(06, 0f, fd, 83, fc, 0e, d9, ce), + be64_to_cpua(a5, 5f, 57, 52, 27, 78, 3a, b5), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, cd), + be64_to_cpua(55, 38, b6, f6, 34, 65, c7, bd), + be64_to_cpua(1c, 57, 56, 8f, 12, b7, 1d, 91), + be64_to_cpua(03, 42, 02, 5f, 50, f0, a2, 0d), + be64_to_cpua(fa, 10, dd, 9b, fb, 36, 1a, 31), + be64_to_cpua(e0, 87, 2c, 44, 4b, 5a, ee, af), + be64_to_cpua(a9, 79, 24, b9, 37, 35, dd, a0), + be64_to_cpua(6b, 35, ae, 65, b5, 99, 12, 0a), + be64_to_cpua(50, 85, 38, f9, 15, 83, 18, 04), + be64_to_cpua(00, 00, 00, 00, 00, 00, 01, cf) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, + { + .key = /* secp521r1(sha384) */ + "\x04\x00\x2e\xd6\x21\x04\x75\xc3\xdc\x7d\xff\x0e\xf3\x70\x25\x2b" + "\xad\x72\xfc\x5a\x91\xf1\xd5\x9c\x64\xf3\x1f\x47\x11\x10\x62\x33" + "\xfd\x2e\xe8\x32\xca\x9e\x6f\x0a\x4c\x5b\x35\x9a\x46\xc5\xe7\xd4" + "\x38\xda\xb2\xf0\xf4\x87\xf3\x86\xf4\xea\x70\xad\x1e\xd4\x78\x8c" + "\x36\x18\x17\x00\xa2\xa0\x34\x1b\x2e\x6a\xdf\x06\xd6\x99\x2d\x47" + "\x50\x92\x1a\x8a\x72\x9c\x23\x44\xfa\xa7\xa9\xed\xa6\xef\x26\x14" + "\xb3\x9d\xfe\x5e\xa3\x8c\xd8\x29\xf8\xdf\xad\xa6\xab\xfc\xdd\x46" + "\x22\x6e\xd7\x35\xc7\x23\xb7\x13\xae\xb6\x34\xff\xd7\x80\xe5\x39" + "\xb3\x3b\x5b\x1b\x94", + .key_len = 133, + .m = + "\x36\x98\xd6\x82\xfa\xad\xed\x3c\xb9\x40\xb6\x4d\x9e\xb7\x04\x26" + "\xad\x72\x34\x44\xd2\x81\xb4\x9b\xbe\x01\x04\x7a\xd8\x50\xf8\x59" + "\xba\xad\x23\x85\x6b\x59\xbe\xfb\xf6\x86\xd4\x67\xa8\x43\x28\x76", + .m_size = 48, + .c = (const unsigned char[]){ + be64_to_cpua(b8, 6a, dd, fb, e6, 63, 4e, 28), + be64_to_cpua(84, 59, fd, 1a, c4, 40, dd, 43), + be64_to_cpua(32, 76, 06, d0, f9, c0, e4, e6), + be64_to_cpua(e4, df, 9b, 7d, 9e, 47, ca, 33), + be64_to_cpua(7e, 42, 71, 86, 57, 2d, f1, 7d), + be64_to_cpua(f2, 4b, 64, 98, f7, ec, da, c7), + be64_to_cpua(ec, 51, dc, e8, 35, 5e, ae, 16), + be64_to_cpua(96, 76, 3c, 27, ea, aa, 9c, 26), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, 93), + be64_to_cpua(c6, 4f, ab, 2b, 62, c1, 42, b1), + be64_to_cpua(e5, 5a, 94, 56, cf, 8f, b4, 22), + be64_to_cpua(6a, c3, f3, 7a, d1, fa, e7, a7), + be64_to_cpua(df, c4, c0, db, 54, db, 8a, 0d), + be64_to_cpua(da, a7, cd, 26, 28, 76, 3b, 52), + be64_to_cpua(e4, 3c, bc, 93, 65, 57, 1c, 30), + be64_to_cpua(55, ce, 37, 97, c9, 05, 51, e5), + be64_to_cpua(c3, 6a, 87, 6e, b5, 13, 1f, 20), + be64_to_cpua(00, 00, 00, 00, 00, 00, 00, ff) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, + { + .key = /* secp521r1(sha512) */ + "\x04\x00\xc7\x65\xee\x0b\x86\x7d\x8f\x02\xf1\x74\x5b\xb0\x4c\x3f" + "\xa6\x35\x60\x9f\x55\x23\x11\xcc\xdf\xb8\x42\x99\xee\x6c\x96\x6a" + "\x27\xa2\x56\xb2\x2b\x03\xad\x0f\xe7\x97\xde\x09\x5d\xb4\xc5\x5f" + "\xbd\x87\x37\xbf\x5a\x16\x35\x56\x08\xfd\x6f\x06\x1a\x1c\x84\xee" + "\xc3\x64\xb3\x00\x9e\xbd\x6e\x60\x76\xee\x69\xfd\x3a\xb8\xcd\x7e" + "\x91\x68\x53\x57\x44\x13\x2e\x77\x09\x2a\xbe\x48\xbd\x91\xd8\xf6" + "\x21\x16\x53\x99\xd5\xf0\x40\xad\xa6\xf8\x58\x26\xb6\x9a\xf8\x77" + "\xfe\x3a\x05\x1a\xdb\xa9\x0f\xc0\x6c\x76\x30\x8c\xd8\xde\x44\xae" + "\xd0\x17\xdf\x49\x6a", + .key_len = 133, + .m = + "\x5c\xa6\xbc\x79\xb8\xa0\x1e\x11\x83\xf7\xe9\x05\xdf\xba\xf7\x69" + "\x97\x22\x32\xe4\x94\x7c\x65\xbd\x74\xc6\x9a\x8b\xbd\x0d\xdc\xed" + "\xf5\x9c\xeb\xe1\xc5\x68\x40\xf2\xc7\x04\xde\x9e\x0d\x76\xc5\xa3" + "\xf9\x3c\x6c\x98\x08\x31\xbd\x39\xe8\x42\x7f\x80\x39\x6f\xfe\x68", + .m_size = 64, + .c = (const unsigned char[]){ + be64_to_cpua(28, b5, 04, b0, b6, 33, 1c, 7e), + be64_to_cpua(80, a6, 13, fc, b6, 90, f7, bb), + be64_to_cpua(27, 93, e8, 6c, 49, 7d, 28, fc), + be64_to_cpua(1f, 12, 3e, b7, 7e, 51, ff, 7f), + be64_to_cpua(fb, 62, 1e, 42, 03, 6c, 74, 8a), + be64_to_cpua(63, 0e, 02, cc, 94, a9, 05, b9), + be64_to_cpua(aa, 86, ec, a8, 05, 03, 52, 56), + be64_to_cpua(71, 86, 96, ac, 21, 33, 7e, 4e), + be64_to_cpua(00, 00, 00, 00, 00, 00, 01, 5c), + be64_to_cpua(46, 1e, 77, 44, 78, e0, d1, 04), + be64_to_cpua(72, 74, 13, 63, 39, a6, e5, 25), + be64_to_cpua(00, 55, bb, 6a, b4, 73, 00, d2), + be64_to_cpua(71, d0, e9, ca, a7, c0, cb, aa), + be64_to_cpua(7a, 76, 37, 51, 47, 49, 98, 12), + be64_to_cpua(88, 05, 3e, 43, 39, 01, bd, b7), + be64_to_cpua(95, 35, 89, 4f, 41, 5f, 9e, 19), + be64_to_cpua(43, 52, 1d, e3, c6, bd, 5a, 40), + be64_to_cpua(00, 00, 00, 00, 00, 00, 01, 70) }, + .c_size = ECC_MAX_BYTES * 2, + .public_key_vec = true, + }, +}; + +/* + * ECDSA X9.62 test vectors. + * + * Identical to ECDSA test vectors, except signature in "c" is X9.62 encoded. + */ +static const struct sig_testvec x962_ecdsa_nist_p192_tv_template[] = { + { + .key = /* secp192r1(sha1) */ "\x04\xf7\x46\xf8\x2f\x15\xf6\x22\x8e\xd7\x57\x4f\xcc\xe7\xbb\xc1" "\xd4\x09\x73\xcf\xea\xd0\x15\x07\x3d\xa5\x8a\x8a\x95\x43\xe4\x68" "\xea\xc6\x25\xc1\xc1\x01\x25\x4c\x7e\xc3\x3c\xa6\x04\x0a\xe7\x08" "\x98", .key_len = 49, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x01", - .param_len = 21, .m = "\xcd\xb9\xd2\x1c\xb7\x6f\xcd\x44\xb3\xfd\x63\xea\xa3\x66\x7f\xae" "\x63\x85\xe7\x82", .m_size = 20, - .algo = OID_id_ecdsa_with_sha1, .c = "\x30\x35\x02\x19\x00\xba\xe5\x93\x83\x6e\xb6\x3b\x63\xa0\x27\x91" "\xc6\xf6\x7f\xc3\x09\xad\x59\xad\x88\x27\xd6\x92\x6b\x02\x18\x10" @@ -674,23 +1371,17 @@ static const struct akcipher_testvec ecdsa_nist_p192_tv_template[] = { "\x80\x6f\xa5\x79\x77\xda\xd0", .c_size = 55, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp192r1(sha224) */ "\x04\xb6\x4b\xb1\xd1\xac\xba\x24\x8f\x65\xb2\x60\x00\x90\xbf\xbd" "\x78\x05\x73\xe9\x79\x1d\x6f\x7c\x0b\xd2\xc3\x93\xa7\x28\xe1\x75" "\xf7\xd5\x95\x1d\x28\x10\xc0\x75\x50\x5c\x1a\x4f\x3f\x8f\xa5\xee" "\xa3", .key_len = 49, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x01", - .param_len = 21, .m = "\x8d\xd6\xb8\x3e\xe5\xff\x23\xf6\x25\xa2\x43\x42\x74\x45\xa7\x40" "\x3a\xff\x2f\xe1\xd3\xf6\x9f\xe8\x33\xcb\x12\x11", .m_size = 28, - .algo = OID_id_ecdsa_with_sha224, .c = "\x30\x34\x02\x18\x5a\x8b\x82\x69\x7e\x8a\x0a\x09\x14\xf8\x11\x2b" "\x55\xdc\xae\x37\x83\x7b\x12\xe6\xb6\x5b\xcb\xd4\x02\x18\x6a\x14" @@ -698,23 +1389,17 @@ static const struct akcipher_testvec ecdsa_nist_p192_tv_template[] = { "\x5c\x99\xdb\x92\x5b\x36", .c_size = 54, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp192r1(sha256) */ "\x04\xe2\x51\x24\x9b\xf7\xb6\x32\x82\x39\x66\x3d\x5b\xec\x3b\xae" "\x0c\xd5\xf2\x67\xd1\xc7\xe1\x02\xe4\xbf\x90\x62\xb8\x55\x75\x56" "\x69\x20\x5e\xcb\x4e\xca\x33\xd6\xcb\x62\x6b\x94\xa9\xa2\xe9\x58" "\x91", .key_len = 49, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x01", - .param_len = 21, .m = "\x35\xec\xa1\xa0\x9e\x14\xde\x33\x03\xb6\xf6\xbd\x0c\x2f\xb2\xfd" "\x1f\x27\x82\xa5\xd7\x70\x3f\xef\xa0\x82\x69\x8e\x73\x31\x8e\xd7", .m_size = 32, - .algo = OID_id_ecdsa_with_sha256, .c = "\x30\x35\x02\x18\x3f\x72\x3f\x1f\x42\xd2\x3f\x1d\x6b\x1a\x58\x56" "\xf1\x8f\xf7\xfd\x01\x48\xfb\x5f\x72\x2a\xd4\x8f\x02\x19\x00\xb3" @@ -722,24 +1407,18 @@ static const struct akcipher_testvec ecdsa_nist_p192_tv_template[] = { "\x3a\x97\xd9\xcd\x1a\x6a\x49", .c_size = 55, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp192r1(sha384) */ "\x04\x5a\x13\xfe\x68\x86\x4d\xf4\x17\xc7\xa4\xe5\x8c\x65\x57\xb7" "\x03\x73\x26\x57\xfb\xe5\x58\x40\xd8\xfd\x49\x05\xab\xf1\x66\x1f" "\xe2\x9d\x93\x9e\xc2\x22\x5a\x8b\x4f\xf3\x77\x22\x59\x7e\xa6\x4e" "\x8b", .key_len = 49, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x01", - .param_len = 21, .m = "\x9d\x2e\x1a\x8f\xed\x6c\x4b\x61\xae\xac\xd5\x19\x79\xce\x67\xf9" "\xa0\x34\xeb\xb0\x81\xf9\xd9\xdc\x6e\xb3\x5c\xa8\x69\xfc\x8a\x61" "\x39\x81\xfb\xfd\x5c\x30\x6b\xa8\xee\xed\x89\xaf\xa3\x05\xe4\x78", .m_size = 48, - .algo = OID_id_ecdsa_with_sha384, .c = "\x30\x35\x02\x19\x00\xf0\xa3\x38\xce\x2b\xf8\x9d\x1a\xcf\x7f\x34" "\xb4\xb4\xe5\xc5\x00\xdd\x15\xbb\xd6\x8c\xa7\x03\x78\x02\x18\x64" @@ -747,25 +1426,19 @@ static const struct akcipher_testvec ecdsa_nist_p192_tv_template[] = { "\x12\x3b\x3b\x28\xfb\x6d\xe1", .c_size = 55, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp192r1(sha512) */ "\x04\xd5\xf2\x6e\xc3\x94\x5c\x52\xbc\xdf\x86\x6c\x14\xd1\xca\xea" "\xcc\x72\x3a\x8a\xf6\x7a\x3a\x56\x36\x3b\xca\xc6\x94\x0e\x17\x1d" "\x9e\xa0\x58\x28\xf9\x4b\xe6\xd1\xa5\x44\x91\x35\x0d\xe7\xf5\x11" "\x57", .key_len = 49, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x01", - .param_len = 21, .m = "\xd5\x4b\xe9\x36\xda\xd8\x6e\xc0\x50\x03\xbe\x00\x43\xff\xf0\x23" "\xac\xa2\x42\xe7\x37\x77\x79\x52\x8f\x3e\xc0\x16\xc1\xfc\x8c\x67" "\x16\xbc\x8a\x5d\x3b\xd3\x13\xbb\xb6\xc0\x26\x1b\xeb\x33\xcc\x70" "\x4a\xf2\x11\x37\xe8\x1b\xba\x55\xac\x69\xe1\x74\x62\x7c\x6e\xb5", .m_size = 64, - .algo = OID_id_ecdsa_with_sha512, .c = "\x30\x35\x02\x19\x00\x88\x5b\x8f\x59\x43\xbf\xcf\xc6\xdd\x3f\x07" "\x87\x12\xa0\xd4\xac\x2b\x11\x2d\x1c\xb6\x06\xc9\x6c\x02\x18\x73" @@ -773,28 +1446,22 @@ static const struct akcipher_testvec ecdsa_nist_p192_tv_template[] = { "\x6a\xdf\x97\xfd\x82\x76\x24", .c_size = 55, .public_key_vec = true, - .siggen_sigver_test = true, }, }; -static const struct akcipher_testvec ecdsa_nist_p256_tv_template[] = { +static const struct sig_testvec x962_ecdsa_nist_p256_tv_template[] = { { - .key = + .key = /* secp256r1(sha1) */ "\x04\xb9\x7b\xbb\xd7\x17\x64\xd2\x7e\xfc\x81\x5d\x87\x06\x83\x41" "\x22\xd6\x9a\xaa\x87\x17\xec\x4f\x63\x55\x2f\x94\xba\xdd\x83\xe9" "\x34\x4b\xf3\xe9\x91\x13\x50\xb6\xcb\xca\x62\x08\xe7\x3b\x09\xdc" "\xc3\x63\x4b\x2d\xb9\x73\x53\xe4\x45\xe6\x7c\xad\xe7\x6b\xb0\xe8" "\xaf", .key_len = 65, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x07", - .param_len = 21, .m = "\xc2\x2b\x5f\x91\x78\x34\x26\x09\x42\x8d\x6f\x51\xb2\xc5\xaf\x4c" "\x0b\xde\x6a\x42", .m_size = 20, - .algo = OID_id_ecdsa_with_sha1, .c = "\x30\x46\x02\x21\x00\xf9\x25\xce\x9f\x3a\xa6\x35\x81\xcf\xd4\xe7" "\xb7\xf0\x82\x56\x41\xf7\xd4\xad\x8d\x94\x5a\x69\x89\xee\xca\x6a" @@ -803,24 +1470,18 @@ static const struct akcipher_testvec ecdsa_nist_p256_tv_template[] = { "\xfb\x9d\x8b\xde\xd4\x8d\x6f\xad", .c_size = 72, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp256r1(sha224) */ "\x04\x8b\x6d\xc0\x33\x8e\x2d\x8b\x67\xf5\xeb\xc4\x7f\xa0\xf5\xd9" "\x7b\x03\xa5\x78\x9a\xb5\xea\x14\xe4\x23\xd0\xaf\xd7\x0e\x2e\xa0" "\xc9\x8b\xdb\x95\xf8\xb3\xaf\xac\x00\x2c\x2c\x1f\x7a\xfd\x95\x88" "\x43\x13\xbf\xf3\x1c\x05\x1a\x14\x18\x09\x3f\xd6\x28\x3e\xc5\xa0" "\xd4", .key_len = 65, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x07", - .param_len = 21, .m = "\x1a\x15\xbc\xa3\xe4\xed\x3a\xb8\x23\x67\xc6\xc4\x34\xf8\x6c\x41" "\x04\x0b\xda\xc5\x77\xfa\x1c\x2d\xe6\x2c\x3b\xe0", .m_size = 28, - .algo = OID_id_ecdsa_with_sha224, .c = "\x30\x44\x02\x20\x20\x43\xfa\xc0\x9f\x9d\x7b\xe7\xae\xce\x77\x59" "\x1a\xdb\x59\xd5\x34\x62\x79\xcb\x6a\x91\x67\x2e\x7d\x25\xd8\x25" @@ -829,24 +1490,18 @@ static const struct akcipher_testvec ecdsa_nist_p256_tv_template[] = { "\x2e\x8b\xde\x5a\x04\x0e", .c_size = 70, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp256r1(sha256) */ "\x04\xf1\xea\xc4\x53\xf3\xb9\x0e\x9f\x7e\xad\xe3\xea\xd7\x0e\x0f" "\xd6\x98\x9a\xca\x92\x4d\x0a\x80\xdb\x2d\x45\xc7\xec\x4b\x97\x00" "\x2f\xe9\x42\x6c\x29\xdc\x55\x0e\x0b\x53\x12\x9b\x2b\xad\x2c\xe9" "\x80\xe6\xc5\x43\xc2\x1d\x5e\xbb\x65\x21\x50\xb6\x37\xb0\x03\x8e" "\xb8", .key_len = 65, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x07", - .param_len = 21, .m = "\x8f\x43\x43\x46\x64\x8f\x6b\x96\xdf\x89\xdd\xa9\x01\xc5\x17\x6b" "\x10\xa6\xd8\x39\x61\xdd\x3c\x1a\xc8\x8b\x59\xb2\xdc\x32\x7a\xa4", .m_size = 32, - .algo = OID_id_ecdsa_with_sha256, .c = "\x30\x45\x02\x20\x08\x31\xfa\x74\x0d\x1d\x21\x5d\x09\xdc\x29\x63" "\xa8\x1a\xad\xfc\xac\x44\xc3\xe8\x24\x11\x2d\xa4\x91\xdc\x02\x67" @@ -855,25 +1510,19 @@ static const struct akcipher_testvec ecdsa_nist_p256_tv_template[] = { "\x2a\x65\x35\x23\xe3\x1d\xfa", .c_size = 71, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp256r1(sha384) */ "\x04\xc5\xc6\xea\x60\xc9\xce\xad\x02\x8d\xf5\x3e\x24\xe3\x52\x1d" "\x28\x47\x3b\xc3\x6b\xa4\x99\x35\x99\x11\x88\x88\xc8\xf4\xee\x7e" "\x8c\x33\x8f\x41\x03\x24\x46\x2b\x1a\x82\xf9\x9f\xe1\x97\x1b\x00" "\xda\x3b\x24\x41\xf7\x66\x33\x58\x3d\x3a\x81\xad\xcf\x16\xe9\xe2" "\x7c", .key_len = 65, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x07", - .param_len = 21, .m = "\x3e\x78\x70\xfb\xcd\x66\xba\x91\xa1\x79\xff\x1e\x1c\x6b\x78\xe6" "\xc0\x81\x3a\x65\x97\x14\x84\x36\x14\x1a\x9a\xb7\xc5\xab\x84\x94" "\x5e\xbb\x1b\x34\x71\xcb\x41\xe1\xf6\xfc\x92\x7b\x34\xbb\x86\xbb", .m_size = 48, - .algo = OID_id_ecdsa_with_sha384, .c = "\x30\x46\x02\x21\x00\x8e\xf3\x6f\xdc\xf8\x69\xa6\x2e\xd0\x2e\x95" "\x54\xd1\x95\x64\x93\x08\xb2\x6b\x24\x94\x48\x46\x5e\xf2\xe4\x6c" @@ -882,26 +1531,20 @@ static const struct akcipher_testvec ecdsa_nist_p256_tv_template[] = { "\xc0\x60\x11\x92\xdc\x17\x89\x12", .c_size = 72, .public_key_vec = true, - .siggen_sigver_test = true, }, { - .key = + .key = /* secp256r1(sha512) */ "\x04\xd7\x27\x46\x49\xf6\x26\x85\x12\x40\x76\x8e\xe2\xe6\x2a\x7a" "\x83\xb1\x4e\x7a\xeb\x3b\x5c\x67\x4a\xb5\xa4\x92\x8c\x69\xff\x38" "\xee\xd9\x4e\x13\x29\x59\xad\xde\x6b\xbb\x45\x31\xee\xfd\xd1\x1b" "\x64\xd3\xb5\xfc\xaf\x9b\x4b\x88\x3b\x0e\xb7\xd6\xdf\xf1\xd5\x92" "\xbf", .key_len = 65, - .params = - "\x30\x13\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x08\x2a\x86\x48" - "\xce\x3d\x03\x01\x07", - .param_len = 21, .m = "\x57\xb7\x9e\xe9\x05\x0a\x8c\x1b\xc9\x13\xe5\x4a\x24\xc7\xe2\xe9" "\x43\xc3\xd1\x76\x62\xf4\x98\x1a\x9c\x13\xb0\x20\x1b\xe5\x39\xca" "\x4f\xd9\x85\x34\x95\xa2\x31\xbc\xbb\xde\xdd\x76\xbb\x61\xe3\xcf" "\x9d\xc0\x49\x7a\xf3\x7a\xc4\x7d\xa8\x04\x4b\x8d\xb4\x4d\x5b\xd6", .m_size = 64, - .algo = OID_id_ecdsa_with_sha512, .c = "\x30\x45\x02\x21\x00\xb8\x6d\x87\x81\x43\xdf\xfb\x9f\x40\xea\x44" "\x81\x00\x4e\x29\x08\xed\x8c\x73\x30\x6c\x22\xb3\x97\x76\xf6\x04" @@ -910,11 +1553,10 @@ static const struct akcipher_testvec ecdsa_nist_p256_tv_template[] = { "\x31\x79\x4a\xe9\x81\x6a\xee", .c_size = 71, .public_key_vec = true, - .siggen_sigver_test = true, }, }; -static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { +static const struct sig_testvec x962_ecdsa_nist_p384_tv_template[] = { { .key = /* secp384r1(sha1) */ "\x04\x89\x25\xf3\x97\x88\xcb\xb0\x78\xc5\x72\x9a\x14\x6e\x7a\xb1" @@ -925,15 +1567,10 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\x0b\x25\xd6\x80\x5c\x3b\xe6\x1a\x98\x48\x91\x45\x7a\x73\xb0\xc3" "\xf1", .key_len = 97, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x22", - .param_len = 18, .m = "\x12\x55\x28\xf0\x77\xd5\xb6\x21\x71\x32\x48\xcd\x28\xa8\x25\x22" "\x3a\x69\xc1\x93", .m_size = 20, - .algo = OID_id_ecdsa_with_sha1, .c = "\x30\x66\x02\x31\x00\xf5\x0f\x24\x4c\x07\x93\x6f\x21\x57\x55\x07" "\x20\x43\x30\xde\xa0\x8d\x26\x8e\xae\x63\x3f\xbc\x20\x3a\xc6\xf1" @@ -944,7 +1581,6 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\x79\x12\x2a\xb7\xc5\x15\x92\xc5", .c_size = 104, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = /* secp384r1(sha224) */ "\x04\x69\x6c\xcf\x62\xee\xd0\x0d\xe5\xb5\x2f\x70\x54\xcf\x26\xa0" @@ -955,15 +1591,10 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\x6b\x93\x99\x6c\x66\x4c\x42\x3f\x65\x60\x6c\x1c\x0b\x93\x9b\x9d" "\xe0", .key_len = 97, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x22", - .param_len = 18, .m = "\x12\x80\xb6\xeb\x25\xe2\x3d\xf0\x21\x32\x96\x17\x3a\x38\x39\xfd" "\x1f\x05\x34\x7b\xb8\xf9\x71\x66\x03\x4f\xd5\xe5", .m_size = 28, - .algo = OID_id_ecdsa_with_sha224, .c = "\x30\x66\x02\x31\x00\x8a\x51\x84\xce\x13\x1e\xd2\xdc\xec\xcb\xe4" "\x89\x47\xb2\xf7\xbc\x97\xf1\xc8\x72\x26\xcf\x5a\x5e\xc5\xda\xb4" @@ -974,7 +1605,6 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\x88\x2b\x82\x26\x5e\x1c\xda\xfb", .c_size = 104, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = /* secp384r1(sha256) */ "\x04\xee\xd6\xda\x3e\x94\x90\x00\x27\xed\xf8\x64\x55\xd6\x51\x9a" @@ -985,15 +1615,10 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\x17\xc3\x34\x29\xd6\x40\xea\x5c\xb9\x3f\xfb\x32\x2e\x12\x33\xbc" "\xab", .key_len = 97, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x22", - .param_len = 18, .m = "\xaa\xe7\xfd\x03\x26\xcb\x94\x71\xe4\xce\x0f\xc5\xff\xa6\x29\xa3" "\xe1\xcc\x4c\x35\x4e\xde\xca\x80\xab\x26\x0c\x25\xe6\x68\x11\xc2", .m_size = 32, - .algo = OID_id_ecdsa_with_sha256, .c = "\x30\x64\x02\x30\x08\x09\x12\x9d\x6e\x96\x64\xa6\x8e\x3f\x7e\xce" "\x0a\x9b\xaa\x59\xcc\x47\x53\x87\xbc\xbd\x83\x3f\xaf\x06\x3f\x84" @@ -1004,7 +1629,6 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\xf4\x1f\x39\xca\x4d\x43", .c_size = 102, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = /* secp384r1(sha384) */ "\x04\x3a\x2f\x62\xe7\x1a\xcf\x24\xd0\x0b\x7c\xe0\xed\x46\x0a\x4f" @@ -1015,16 +1639,11 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\x21\x67\xe5\x1b\x5a\x52\x31\x68\xd6\xee\xf0\x19\xb0\x55\xed\x89" "\x9e", .key_len = 97, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x22", - .param_len = 18, .m = "\x8d\xf2\xc0\xe9\xa8\xf3\x8e\x44\xc4\x8c\x1a\xa0\xb8\xd7\x17\xdf" "\xf2\x37\x1b\xc6\xe3\xf5\x62\xcc\x68\xf5\xd5\x0b\xbf\x73\x2b\xb1" "\xb0\x4c\x04\x00\x31\xab\xfe\xc8\xd6\x09\xc8\xf2\xea\xd3\x28\xff", .m_size = 48, - .algo = OID_id_ecdsa_with_sha384, .c = "\x30\x66\x02\x31\x00\x9b\x28\x68\xc0\xa1\xea\x8c\x50\xee\x2e\x62" "\x35\x46\xfa\x00\xd8\x2d\x7a\x91\x5f\x49\x2d\x22\x08\x29\xe6\xfb" @@ -1035,7 +1654,6 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\xab\x8d\x4e\xde\xe6\x6d\x9b\x66", .c_size = 104, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = /* secp384r1(sha512) */ "\x04\xb4\xe7\xc1\xeb\x64\x25\x22\x46\xc3\x86\x61\x80\xbe\x1e\x46" @@ -1046,17 +1664,12 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\xdf\x42\x5c\xc2\x5a\xc7\x0c\xf4\x15\xf7\x1b\xa3\x2e\xd7\x00\xac" "\xa3", .key_len = 97, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x22", - .param_len = 18, .m = "\xe8\xb7\x52\x7d\x1a\x44\x20\x05\x53\x6b\x3a\x68\xf2\xe7\x6c\xa1" "\xae\x9d\x84\xbb\xba\x52\x43\x3e\x2c\x42\x78\x49\xbf\x78\xb2\x71" "\xeb\xe1\xe0\xe8\x42\x7b\x11\xad\x2b\x99\x05\x1d\x36\xe6\xac\xfc" "\x55\x73\xf0\x15\x63\x39\xb8\x6a\x6a\xc5\x91\x5b\xca\x6a\xa8\x0e", .m_size = 64, - .algo = OID_id_ecdsa_with_sha512, .c = "\x30\x63\x02\x2f\x1d\x20\x94\x77\xfe\x31\xfa\x4d\xc6\xef\xda\x02" "\xe7\x0f\x52\x9a\x02\xde\x93\xe8\x83\xe4\x84\x4c\xfc\x6f\x80\xe3" @@ -1067,11 +1680,10 @@ static const struct akcipher_testvec ecdsa_nist_p384_tv_template[] = { "\x3c\x93\xff\x50\x5d", .c_size = 101, .public_key_vec = true, - .siggen_sigver_test = true, }, }; -static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { +static const struct sig_testvec x962_ecdsa_nist_p521_tv_template[] = { { .key = /* secp521r1(sha224) */ "\x04\x01\x4f\x43\x18\xb6\xa9\xc9\x5d\x68\xd3\xa9\x42\xf8\x98\xc0" @@ -1084,15 +1696,10 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\xed\x37\x0f\x99\x3f\x26\xba\xa3\x8e\xff\x79\x34\x7c\x3a\xfe\x1f" "\x3b\x83\x82\x2f\x14", .key_len = 133, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x23", - .param_len = 18, .m = "\xa2\x3a\x6a\x8c\x7b\x3c\xf2\x51\xf8\xbe\x5f\x4f\x3b\x15\x05\xc4" "\xb5\xbc\x19\xe7\x21\x85\xe9\x23\x06\x33\x62\xfb", .m_size = 28, - .algo = OID_id_ecdsa_with_sha224, .c = "\x30\x81\x86\x02\x41\x01\xd6\x43\xe7\xff\x42\xb2\xba\x74\x35\xf6" "\xdc\x6d\x02\x7b\x22\xac\xe2\xef\x07\x92\xee\x60\x94\x06\xf8\x3f" @@ -1105,7 +1712,6 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\xa3\x50\xb1\xa5\x98\x92\x2a\xa5\x52", .c_size = 137, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = /* secp521r1(sha256) */ @@ -1119,15 +1725,10 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\x8a\xe9\x53\xa8\xcf\xce\x43\x0e\x82\x20\x86\xbc\x88\x9c\xb7\xe3" "\xe6\x77\x1e\x1f\x8a", .key_len = 133, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x23", - .param_len = 18, .m = "\xcc\x97\x73\x0c\x73\xa2\x53\x2b\xfa\xd7\x83\x1d\x0c\x72\x1b\x39" "\x80\x71\x8d\xdd\xc5\x9b\xff\x55\x32\x98\x25\xa2\x58\x2e\xb7\x73", .m_size = 32, - .algo = OID_id_ecdsa_with_sha256, .c = "\x30\x81\x88\x02\x42\x00\xcd\xa5\x5f\x57\x52\x27\x78\x3a\xb5\x06" "\x0f\xfd\x83\xfc\x0e\xd9\xce\x50\x9f\x7d\x1f\xca\x8b\xa8\x2d\x56" @@ -1140,7 +1741,6 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\xb7\x1d\x91\x55\x38\xb6\xf6\x34\x65\xc7\xbd", .c_size = 139, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = /* secp521r1(sha384) */ @@ -1154,16 +1754,11 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\x22\x6e\xd7\x35\xc7\x23\xb7\x13\xae\xb6\x34\xff\xd7\x80\xe5\x39" "\xb3\x3b\x5b\x1b\x94", .key_len = 133, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x23", - .param_len = 18, .m = "\x36\x98\xd6\x82\xfa\xad\xed\x3c\xb9\x40\xb6\x4d\x9e\xb7\x04\x26" "\xad\x72\x34\x44\xd2\x81\xb4\x9b\xbe\x01\x04\x7a\xd8\x50\xf8\x59" "\xba\xad\x23\x85\x6b\x59\xbe\xfb\xf6\x86\xd4\x67\xa8\x43\x28\x76", .m_size = 48, - .algo = OID_id_ecdsa_with_sha384, .c = "\x30\x81\x88\x02\x42\x00\x93\x96\x76\x3c\x27\xea\xaa\x9c\x26\xec" "\x51\xdc\xe8\x35\x5e\xae\x16\xf2\x4b\x64\x98\xf7\xec\xda\xc7\x7e" @@ -1176,7 +1771,6 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\x8f\xb4\x22\xc6\x4f\xab\x2b\x62\xc1\x42\xb1", .c_size = 139, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = /* secp521r1(sha512) */ @@ -1190,17 +1784,12 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\xfe\x3a\x05\x1a\xdb\xa9\x0f\xc0\x6c\x76\x30\x8c\xd8\xde\x44\xae" "\xd0\x17\xdf\x49\x6a", .key_len = 133, - .params = - "\x30\x10\x06\x07\x2a\x86\x48\xce\x3d\x02\x01\x06\x05\x2b\x81\x04" - "\x00\x23", - .param_len = 18, .m = "\x5c\xa6\xbc\x79\xb8\xa0\x1e\x11\x83\xf7\xe9\x05\xdf\xba\xf7\x69" "\x97\x22\x32\xe4\x94\x7c\x65\xbd\x74\xc6\x9a\x8b\xbd\x0d\xdc\xed" "\xf5\x9c\xeb\xe1\xc5\x68\x40\xf2\xc7\x04\xde\x9e\x0d\x76\xc5\xa3" "\xf9\x3c\x6c\x98\x08\x31\xbd\x39\xe8\x42\x7f\x80\x39\x6f\xfe\x68", .m_size = 64, - .algo = OID_id_ecdsa_with_sha512, .c = "\x30\x81\x88\x02\x42\x01\x5c\x71\x86\x96\xac\x21\x33\x7e\x4e\xaa" "\x86\xec\xa8\x05\x03\x52\x56\x63\x0e\x02\xcc\x94\xa9\x05\xb9\xfb" @@ -1213,14 +1802,41 @@ static const struct akcipher_testvec ecdsa_nist_p521_tv_template[] = { "\xa6\xe5\x25\x46\x1e\x77\x44\x78\xe0\xd1\x04", .c_size = 139, .public_key_vec = true, - .siggen_sigver_test = true, + }, +}; + +/* + * ECDSA P1363 test vectors. + * + * Identical to ECDSA test vectors, except signature in "c" is P1363 encoded. + */ +static const struct sig_testvec p1363_ecdsa_nist_p256_tv_template[] = { + { + .key = /* secp256r1(sha256) */ + "\x04\xf1\xea\xc4\x53\xf3\xb9\x0e\x9f\x7e\xad\xe3\xea\xd7\x0e\x0f" + "\xd6\x98\x9a\xca\x92\x4d\x0a\x80\xdb\x2d\x45\xc7\xec\x4b\x97\x00" + "\x2f\xe9\x42\x6c\x29\xdc\x55\x0e\x0b\x53\x12\x9b\x2b\xad\x2c\xe9" + "\x80\xe6\xc5\x43\xc2\x1d\x5e\xbb\x65\x21\x50\xb6\x37\xb0\x03\x8e" + "\xb8", + .key_len = 65, + .m = + "\x8f\x43\x43\x46\x64\x8f\x6b\x96\xdf\x89\xdd\xa9\x01\xc5\x17\x6b" + "\x10\xa6\xd8\x39\x61\xdd\x3c\x1a\xc8\x8b\x59\xb2\xdc\x32\x7a\xa4", + .m_size = 32, + .c = + "\x08\x31\xfa\x74\x0d\x1d\x21\x5d\x09\xdc\x29\x63\xa8\x1a\xad\xfc" + "\xac\x44\xc3\xe8\x24\x11\x2d\xa4\x91\xdc\x02\x67\xdc\x0c\xd0\x82" + "\xbd\xff\xce\xee\x42\xc3\x97\xff\xf9\xa9\x81\xac\x4a\x50\xd0\x91" + "\x0a\x6e\x1b\xc4\xaf\xe1\x83\xc3\x4f\x2a\x65\x35\x23\xe3\x1d\xfa", + .c_size = 64, + .public_key_vec = true, }, }; /* * EC-RDSA test vectors are generated by gost-engine. */ -static const struct akcipher_testvec ecrdsa_tv_template[] = { +static const struct sig_testvec ecrdsa_tv_template[] = { { .key = "\x04\x40\xd5\xa7\x77\xf9\x26\x2f\x8c\xbd\xcc\xe3\x1f\x01\x94\x05" @@ -1245,7 +1861,6 @@ static const struct akcipher_testvec ecrdsa_tv_template[] = { "\x79\xd2\x76\x64\xa3\xbd\x66\x10\x79\x05\x5a\x06\x42\xec\xb9\xc9", .m_size = 32, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = @@ -1271,7 +1886,6 @@ static const struct akcipher_testvec ecrdsa_tv_template[] = { "\x11\x23\x4a\x70\x43\x52\x7a\x68\x11\x65\x45\x37\xbb\x25\xb7\x40", .m_size = 32, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = @@ -1297,7 +1911,6 @@ static const struct akcipher_testvec ecrdsa_tv_template[] = { "\x9f\x16\xc6\x1c\xb1\x3f\x84\x41\x69\xec\x34\xfd\xf1\xf9\xa3\x39", .m_size = 32, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = @@ -1332,7 +1945,6 @@ static const struct akcipher_testvec ecrdsa_tv_template[] = { "\xa8\xf6\x80\x01\xb9\x27\xac\xd8\x45\x96\x66\xa1\xee\x48\x08\x3f", .m_size = 64, .public_key_vec = true, - .siggen_sigver_test = true, }, { .key = @@ -1367,14 +1979,68 @@ static const struct akcipher_testvec ecrdsa_tv_template[] = { "\x6d\xf4\xd2\x45\xc2\x83\xa0\x42\x95\x05\x9d\x89\x8e\x0a\xca\xcc", .m_size = 64, .public_key_vec = true, - .siggen_sigver_test = true, + }, +}; + +/* + * PKCS#1 RSA test vectors for hash algorithm "none" + * (i.e. the hash in "m" is not prepended by a Full Hash Prefix) + * + * Obtained from: + * https://vcsjones.dev/sometimes-valid-rsa-dotnet/ + * https://gist.github.com/vcsjones/ab4c2327b53ed018eada76b75ef4fd99 + */ +static const struct sig_testvec pkcs1_rsa_none_tv_template[] = { + { + .key = + "\x30\x82\x01\x0a\x02\x82\x01\x01\x00\xa2\x63\x0b\x39\x44\xb8\xbb" + "\x23\xa7\x44\x49\xbb\x0e\xff\xa1\xf0\x61\x0a\x53\x93\xb0\x98\xdb" + "\xad\x2c\x0f\x4a\xc5\x6e\xff\x86\x3c\x53\x55\x0f\x15\xce\x04\x3f" + "\x2b\xfd\xa9\x96\x96\xd9\xbe\x61\x79\x0b\x5b\xc9\x4c\x86\x76\xe5" + "\xe0\x43\x4b\x22\x95\xee\xc2\x2b\x43\xc1\x9f\xd8\x68\xb4\x8e\x40" + "\x4f\xee\x85\x38\xb9\x11\xc5\x23\xf2\x64\x58\xf0\x15\x32\x6f\x4e" + "\x57\xa1\xae\x88\xa4\x02\xd7\x2a\x1e\xcd\x4b\xe1\xdd\x63\xd5\x17" + "\x89\x32\x5b\xb0\x5e\x99\x5a\xa8\x9d\x28\x50\x0e\x17\xee\x96\xdb" + "\x61\x3b\x45\x51\x1d\xcf\x12\x56\x0b\x92\x47\xfc\xab\xae\xf6\x66" + "\x3d\x47\xac\x70\x72\xe7\x92\xe7\x5f\xcd\x10\xb9\xc4\x83\x64\x94" + "\x19\xbd\x25\x80\xe1\xe8\xd2\x22\xa5\xd0\xba\x02\x7a\xa1\x77\x93" + "\x5b\x65\xc3\xee\x17\x74\xbc\x41\x86\x2a\xdc\x08\x4c\x8c\x92\x8c" + "\x91\x2d\x9e\x77\x44\x1f\x68\xd6\xa8\x74\x77\xdb\x0e\x5b\x32\x8b" + "\x56\x8b\x33\xbd\xd9\x63\xc8\x49\x9d\x3a\xc5\xc5\xea\x33\x0b\xd2" + "\xf1\xa3\x1b\xf4\x8b\xbe\xd9\xb3\x57\x8b\x3b\xde\x04\xa7\x7a\x22" + "\xb2\x24\xae\x2e\xc7\x70\xc5\xbe\x4e\x83\x26\x08\xfb\x0b\xbd\xa9" + "\x4f\x99\x08\xe1\x10\x28\x72\xaa\xcd\x02\x03\x01\x00\x01", + .key_len = 270, + .m = + "\x68\xb4\xf9\x26\x34\x31\x25\xdd\x26\x50\x13\x68\xc1\x99\x26\x71" + "\x19\xa2\xde\x81", + .m_size = 20, + .c = + "\x6a\xdb\x39\xe5\x63\xb3\x25\xde\x58\xca\xc3\xf1\x36\x9c\x0b\x36" + "\xb7\xd6\x69\xf9\xba\xa6\x68\x14\x8c\x24\x52\xd3\x25\xa5\xf3\xad" + "\xc9\x47\x44\xde\x06\xd8\x0f\x56\xca\x2d\xfb\x0f\xe9\x99\xe2\x9d" + "\x8a\xe8\x7f\xfb\x9a\x99\x96\xf1\x2c\x4a\xe4\xc0\xae\x4d\x29\x47" + "\x38\x96\x51\x2f\x6d\x8e\xb8\x88\xbd\x1a\x0a\x70\xbc\x23\x38\x67" + "\x62\x22\x01\x23\x71\xe5\xbb\x95\xea\x6b\x8d\x31\x62\xbf\xf0\xc4" + "\xb9\x46\xd6\x67\xfc\x4c\xe6\x1f\xd6\x5d\xf7\xa9\xad\x3a\xf1\xbf" + "\xa2\xf9\x66\xde\xb6\x8e\xec\x8f\x81\x8d\x1e\x3a\x12\x27\x6a\xfc" + "\xae\x92\x9f\xc3\x87\xc3\xba\x8d\x04\xb8\x8f\x0f\x61\x68\x9a\x96" + "\x2c\x80\x2c\x32\x40\xde\x9d\xb9\x9b\xe2\xe4\x45\x2e\x91\x47\x5c" + "\x47\xa4\x9d\x02\x57\x59\xf7\x75\x5d\x5f\x32\x82\x75\x5d\xe5\x78" + "\xc9\x19\x61\x46\x06\x9d\xa5\x1d\xd6\x32\x48\x9a\xdb\x09\x29\x81" + "\x14\x2e\xf0\x27\xe9\x37\x13\x74\xec\xa5\xcd\x67\x6b\x19\xf6\x88" + "\xf0\xc2\x8b\xa8\x7f\x2f\x76\x5a\x3e\x0c\x47\x5d\xe8\x82\x50\x27" + "\x40\xce\x27\x41\x45\xa0\xcf\xaa\x2f\xd3\xad\x3c\xbf\x73\xff\x93" + "\xe3\x78\x49\xd9\xa9\x78\x22\x81\x9a\xe5\xe2\x94\xe9\x40\xab\xf1", + .c_size = 256, + .public_key_vec = true, }, }; /* * PKCS#1 RSA test vectors. Obtained from CAVS testing. */ -static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { +static const struct sig_testvec pkcs1_rsa_tv_template[] = { { .key = "\x30\x82\x04\xa5\x02\x01\x00\x02\x82\x01\x01\x00\xd7\x1e\x77\x82" @@ -1486,7 +2152,6 @@ static const struct akcipher_testvec pkcs1pad_rsa_tv_template[] = { "\xda\x62\x8d\xe1\x2a\x71\x91\x43\x40\x61\x3c\x5a\xbe\x86\xfc\x5b" "\xe6\xf9\xa9\x16\x31\x1f\xaf\x25\x6d\xc2\x4a\x23\x6e\x63\x02\xa2", .c_size = 256, - .siggen_sigver_test = true, } }; diff --git a/drivers/char/hw_random/Kconfig b/drivers/char/hw_random/Kconfig index b51d9e243f35..17854f052386 100644 --- a/drivers/char/hw_random/Kconfig +++ b/drivers/char/hw_random/Kconfig @@ -50,7 +50,7 @@ config HW_RANDOM_INTEL config HW_RANDOM_AMD tristate "AMD HW Random Number Generator support" - depends on (X86 || PPC_MAPLE || COMPILE_TEST) + depends on (X86 || COMPILE_TEST) depends on PCI && HAS_IOPORT_MAP default HW_RANDOM help @@ -62,6 +62,19 @@ config HW_RANDOM_AMD If unsure, say Y. +config HW_RANDOM_AIROHA + tristate "Airoha True HW Random Number Generator support" + depends on ARCH_AIROHA || COMPILE_TEST + default HW_RANDOM + help + This driver provides kernel-side support for the True Random Number + Generator hardware found on Airoha SoC. + + To compile this driver as a module, choose M here: the + module will be called airoha-rng. + + If unsure, say Y. + config HW_RANDOM_ATMEL tristate "Atmel Random Number Generator support" depends on (ARCH_AT91 || COMPILE_TEST) @@ -99,9 +112,22 @@ config HW_RANDOM_BCM2835 If unsure, say Y. +config HW_RANDOM_BCM74110 + tristate "Broadcom BCM74110 Random Number Generator support" + depends on ARCH_BRCMSTB || COMPILE_TEST + default HW_RANDOM + help + This driver provides kernel-side support for the Random Number + Generator hardware found on the Broadcom BCM74110 SoCs. + + To compile this driver as a module, choose M here: the + module will be called bcm74110-rng + + If unsure, say Y. + config HW_RANDOM_IPROC_RNG200 tristate "Broadcom iProc/STB RNG200 support" - depends on ARCH_BCM_IPROC || ARCH_BCM2835 || ARCH_BRCMSTB || COMPILE_TEST + depends on ARCH_BCM_IPROC || ARCH_BCM2835 || ARCH_BCMBCA || ARCH_BRCMSTB || COMPILE_TEST default HW_RANDOM help This driver provides kernel-side support for the RNG200 diff --git a/drivers/char/hw_random/Makefile b/drivers/char/hw_random/Makefile index 01f012eab440..b9132b3f5d21 100644 --- a/drivers/char/hw_random/Makefile +++ b/drivers/char/hw_random/Makefile @@ -8,6 +8,7 @@ rng-core-y := core.o obj-$(CONFIG_HW_RANDOM_TIMERIOMEM) += timeriomem-rng.o obj-$(CONFIG_HW_RANDOM_INTEL) += intel-rng.o obj-$(CONFIG_HW_RANDOM_AMD) += amd-rng.o +obj-$(CONFIG_HW_RANDOM_AIROHA) += airoha-trng.o obj-$(CONFIG_HW_RANDOM_ATMEL) += atmel-rng.o obj-$(CONFIG_HW_RANDOM_BA431) += ba431-rng.o obj-$(CONFIG_HW_RANDOM_GEODE) += geode-rng.o @@ -31,6 +32,7 @@ obj-$(CONFIG_HW_RANDOM_POWERNV) += powernv-rng.o obj-$(CONFIG_HW_RANDOM_HISI) += hisi-rng.o obj-$(CONFIG_HW_RANDOM_HISTB) += histb-rng.o obj-$(CONFIG_HW_RANDOM_BCM2835) += bcm2835-rng.o +obj-$(CONFIG_HW_RANDOM_BCM74110) += bcm74110-rng.o obj-$(CONFIG_HW_RANDOM_IPROC_RNG200) += iproc-rng200.o obj-$(CONFIG_HW_RANDOM_ST) += st-rng.o obj-$(CONFIG_HW_RANDOM_XGENE) += xgene-rng.o diff --git a/drivers/char/hw_random/airoha-trng.c b/drivers/char/hw_random/airoha-trng.c new file mode 100644 index 000000000000..1dbfa9505c21 --- /dev/null +++ b/drivers/char/hw_random/airoha-trng.c @@ -0,0 +1,243 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (C) 2024 Christian Marangi */ + +#include <linux/kernel.h> +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/bitfield.h> +#include <linux/delay.h> +#include <linux/hw_random.h> +#include <linux/interrupt.h> +#include <linux/io.h> +#include <linux/iopoll.h> +#include <linux/platform_device.h> + +#define TRNG_IP_RDY 0x800 +#define CNT_TRANS GENMASK(15, 8) +#define SAMPLE_RDY BIT(0) +#define TRNG_NS_SEK_AND_DAT_EN 0x804 +#define RNG_EN BIT(31) /* referenced as ring_en */ +#define RAW_DATA_EN BIT(16) +#define TRNG_HEALTH_TEST_SW_RST 0x808 +#define SW_RST BIT(0) /* Active High */ +#define TRNG_INTR_EN 0x818 +#define INTR_MASK BIT(16) +#define CONTINUOUS_HEALTH_INITR_EN BIT(2) +#define SW_STARTUP_INITR_EN BIT(1) +#define RST_STARTUP_INITR_EN BIT(0) +/* Notice that Health Test are done only out of Reset and with RNG_EN */ +#define TRNG_HEALTH_TEST_STATUS 0x824 +#define CONTINUOUS_HEALTH_AP_TEST_FAIL BIT(23) +#define CONTINUOUS_HEALTH_RC_TEST_FAIL BIT(22) +#define SW_STARTUP_TEST_DONE BIT(21) +#define SW_STARTUP_AP_TEST_FAIL BIT(20) +#define SW_STARTUP_RC_TEST_FAIL BIT(19) +#define RST_STARTUP_TEST_DONE BIT(18) +#define RST_STARTUP_AP_TEST_FAIL BIT(17) +#define RST_STARTUP_RC_TEST_FAIL BIT(16) +#define RAW_DATA_VALID BIT(7) + +#define TRNG_RAW_DATA_OUT 0x828 + +#define TRNG_CNT_TRANS_VALID 0x80 +#define BUSY_LOOP_SLEEP 10 +#define BUSY_LOOP_TIMEOUT (BUSY_LOOP_SLEEP * 10000) + +struct airoha_trng { + void __iomem *base; + struct hwrng rng; + struct device *dev; + + struct completion rng_op_done; +}; + +static int airoha_trng_irq_mask(struct airoha_trng *trng) +{ + u32 val; + + val = readl(trng->base + TRNG_INTR_EN); + val |= INTR_MASK; + writel(val, trng->base + TRNG_INTR_EN); + + return 0; +} + +static int airoha_trng_irq_unmask(struct airoha_trng *trng) +{ + u32 val; + + val = readl(trng->base + TRNG_INTR_EN); + val &= ~INTR_MASK; + writel(val, trng->base + TRNG_INTR_EN); + + return 0; +} + +static int airoha_trng_init(struct hwrng *rng) +{ + struct airoha_trng *trng = container_of(rng, struct airoha_trng, rng); + int ret; + u32 val; + + val = readl(trng->base + TRNG_NS_SEK_AND_DAT_EN); + val |= RNG_EN; + writel(val, trng->base + TRNG_NS_SEK_AND_DAT_EN); + + /* Set out of SW Reset */ + airoha_trng_irq_unmask(trng); + writel(0, trng->base + TRNG_HEALTH_TEST_SW_RST); + + ret = wait_for_completion_timeout(&trng->rng_op_done, BUSY_LOOP_TIMEOUT); + if (ret <= 0) { + dev_err(trng->dev, "Timeout waiting for Health Check\n"); + airoha_trng_irq_mask(trng); + return -ENODEV; + } + + /* Check if Health Test Failed */ + val = readl(trng->base + TRNG_HEALTH_TEST_STATUS); + if (val & (RST_STARTUP_AP_TEST_FAIL | RST_STARTUP_RC_TEST_FAIL)) { + dev_err(trng->dev, "Health Check fail: %s test fail\n", + val & RST_STARTUP_AP_TEST_FAIL ? "AP" : "RC"); + return -ENODEV; + } + + /* Check if IP is ready */ + ret = readl_poll_timeout(trng->base + TRNG_IP_RDY, val, + val & SAMPLE_RDY, 10, 1000); + if (ret < 0) { + dev_err(trng->dev, "Timeout waiting for IP ready"); + return -ENODEV; + } + + /* CNT_TRANS must be 0x80 for IP to be considered ready */ + ret = readl_poll_timeout(trng->base + TRNG_IP_RDY, val, + FIELD_GET(CNT_TRANS, val) == TRNG_CNT_TRANS_VALID, + 10, 1000); + if (ret < 0) { + dev_err(trng->dev, "Timeout waiting for IP ready"); + return -ENODEV; + } + + return 0; +} + +static void airoha_trng_cleanup(struct hwrng *rng) +{ + struct airoha_trng *trng = container_of(rng, struct airoha_trng, rng); + u32 val; + + val = readl(trng->base + TRNG_NS_SEK_AND_DAT_EN); + val &= ~RNG_EN; + writel(val, trng->base + TRNG_NS_SEK_AND_DAT_EN); + + /* Put it in SW Reset */ + writel(SW_RST, trng->base + TRNG_HEALTH_TEST_SW_RST); +} + +static int airoha_trng_read(struct hwrng *rng, void *buf, size_t max, bool wait) +{ + struct airoha_trng *trng = container_of(rng, struct airoha_trng, rng); + u32 *data = buf; + u32 status; + int ret; + + ret = readl_poll_timeout(trng->base + TRNG_HEALTH_TEST_STATUS, status, + status & RAW_DATA_VALID, 10, 1000); + if (ret < 0) { + dev_err(trng->dev, "Timeout waiting for TRNG RAW Data valid\n"); + return ret; + } + + *data = readl(trng->base + TRNG_RAW_DATA_OUT); + + return 4; +} + +static irqreturn_t airoha_trng_irq(int irq, void *priv) +{ + struct airoha_trng *trng = (struct airoha_trng *)priv; + + airoha_trng_irq_mask(trng); + /* Just complete the task, we will read the value later */ + complete(&trng->rng_op_done); + + return IRQ_HANDLED; +} + +static int airoha_trng_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct airoha_trng *trng; + int irq, ret; + u32 val; + + trng = devm_kzalloc(dev, sizeof(*trng), GFP_KERNEL); + if (!trng) + return -ENOMEM; + + trng->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(trng->base)) + return PTR_ERR(trng->base); + + irq = platform_get_irq(pdev, 0); + if (irq < 0) + return irq; + + airoha_trng_irq_mask(trng); + ret = devm_request_irq(&pdev->dev, irq, airoha_trng_irq, 0, + pdev->name, (void *)trng); + if (ret) { + dev_err(dev, "Can't get interrupt working.\n"); + return ret; + } + + init_completion(&trng->rng_op_done); + + /* Enable interrupt for SW reset Health Check */ + val = readl(trng->base + TRNG_INTR_EN); + val |= RST_STARTUP_INITR_EN; + writel(val, trng->base + TRNG_INTR_EN); + + /* Set output to raw data */ + val = readl(trng->base + TRNG_NS_SEK_AND_DAT_EN); + val |= RAW_DATA_EN; + writel(val, trng->base + TRNG_NS_SEK_AND_DAT_EN); + + /* Put it in SW Reset */ + writel(SW_RST, trng->base + TRNG_HEALTH_TEST_SW_RST); + + trng->dev = dev; + trng->rng.name = pdev->name; + trng->rng.init = airoha_trng_init; + trng->rng.cleanup = airoha_trng_cleanup; + trng->rng.read = airoha_trng_read; + + ret = devm_hwrng_register(dev, &trng->rng); + if (ret) { + dev_err(dev, "failed to register rng device: %d\n", ret); + return ret; + } + + return 0; +} + +static const struct of_device_id airoha_trng_of_match[] = { + { .compatible = "airoha,en7581-trng", }, + {}, +}; +MODULE_DEVICE_TABLE(of, airoha_trng_of_match); + +static struct platform_driver airoha_trng_driver = { + .driver = { + .name = "airoha-trng", + .of_match_table = airoha_trng_of_match, + }, + .probe = airoha_trng_probe, +}; + +module_platform_driver(airoha_trng_driver); + +MODULE_LICENSE("GPL"); +MODULE_AUTHOR("Christian Marangi <ansuelsmth@gmail.com>"); +MODULE_DESCRIPTION("Airoha True Random Number Generator driver"); diff --git a/drivers/char/hw_random/atmel-rng.c b/drivers/char/hw_random/atmel-rng.c index e9157255f851..143406bc6939 100644 --- a/drivers/char/hw_random/atmel-rng.c +++ b/drivers/char/hw_random/atmel-rng.c @@ -216,7 +216,7 @@ MODULE_DEVICE_TABLE(of, atmel_trng_dt_ids); static struct platform_driver atmel_trng_driver = { .probe = atmel_trng_probe, - .remove_new = atmel_trng_remove, + .remove = atmel_trng_remove, .driver = { .name = "atmel-trng", .pm = pm_ptr(&atmel_trng_pm_ops), diff --git a/drivers/char/hw_random/bcm74110-rng.c b/drivers/char/hw_random/bcm74110-rng.c new file mode 100644 index 000000000000..5c64148e91f1 --- /dev/null +++ b/drivers/char/hw_random/bcm74110-rng.c @@ -0,0 +1,125 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copyright (c) 2024 Broadcom + */ + +#define pr_fmt(fmt) KBUILD_MODNAME ": " fmt + +#include <linux/module.h> +#include <linux/mod_devicetable.h> +#include <linux/kernel.h> +#include <linux/io.h> +#include <linux/delay.h> +#include <linux/platform_device.h> +#include <linux/random.h> +#include <linux/hw_random.h> + +#define HOST_REV_ID 0x00 +#define HOST_FIFO_DEPTH 0x04 +#define HOST_FIFO_COUNT 0x08 +#define HOST_FIFO_THRESHOLD 0x0c +#define HOST_FIFO_DATA 0x10 + +#define HOST_FIFO_COUNT_MASK 0xffff + +/* Delay range in microseconds */ +#define FIFO_DELAY_MIN_US 3 +#define FIFO_DELAY_MAX_US 7 +#define FIFO_DELAY_MAX_COUNT 10 + +struct bcm74110_priv { + void __iomem *base; +}; + +static inline int bcm74110_rng_fifo_count(void __iomem *mem) +{ + return readl_relaxed(mem) & HOST_FIFO_COUNT_MASK; +} + +static int bcm74110_rng_read(struct hwrng *rng, void *buf, size_t max, + bool wait) +{ + struct bcm74110_priv *priv = (struct bcm74110_priv *)rng->priv; + void __iomem *fc_addr = priv->base + HOST_FIFO_COUNT; + void __iomem *fd_addr = priv->base + HOST_FIFO_DATA; + unsigned underrun_count = 0; + u32 max_words = max / sizeof(u32); + u32 num_words; + unsigned i; + + /* + * We need to check how many words are available in the RNG FIFO. If + * there aren't any, we need to wait for some to become available. + */ + while ((num_words = bcm74110_rng_fifo_count(fc_addr)) == 0) { + if (!wait) + return 0; + /* + * As a precaution, limit how long we wait. If the FIFO doesn't + * refill within the allotted time, return 0 (=no data) to the + * caller. + */ + if (likely(underrun_count < FIFO_DELAY_MAX_COUNT)) + usleep_range(FIFO_DELAY_MIN_US, FIFO_DELAY_MAX_US); + else + return 0; + underrun_count++; + } + if (num_words > max_words) + num_words = max_words; + + /* Bail early if we run out of random numbers unexpectedly */ + for (i = 0; i < num_words && bcm74110_rng_fifo_count(fc_addr) > 0; i++) + ((u32 *)buf)[i] = readl_relaxed(fd_addr); + + return i * sizeof(u32); +} + +static struct hwrng bcm74110_hwrng = { + .read = bcm74110_rng_read, +}; + +static int bcm74110_rng_probe(struct platform_device *pdev) +{ + struct device *dev = &pdev->dev; + struct bcm74110_priv *priv; + int rc; + + priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); + if (!priv) + return -ENOMEM; + + bcm74110_hwrng.name = pdev->name; + bcm74110_hwrng.priv = (unsigned long)priv; + + priv->base = devm_platform_ioremap_resource(pdev, 0); + if (IS_ERR(priv->base)) + return PTR_ERR(priv->base); + + rc = devm_hwrng_register(dev, &bcm74110_hwrng); + if (rc) + dev_err(dev, "hwrng registration failed (%d)\n", rc); + else + dev_info(dev, "hwrng registered\n"); + + return rc; +} + +static const struct of_device_id bcm74110_rng_match[] = { + { .compatible = "brcm,bcm74110-rng", }, + {}, +}; +MODULE_DEVICE_TABLE(of, bcm74110_rng_match); + +static struct platform_driver bcm74110_rng_driver = { + .driver = { + .name = KBUILD_MODNAME, + .of_match_table = bcm74110_rng_match, + }, + .probe = bcm74110_rng_probe, +}; +module_platform_driver(bcm74110_rng_driver); + +MODULE_AUTHOR("Markus Mayer <mmayer@broadcom.com>"); +MODULE_DESCRIPTION("BCM 74110 Random Number Generator (RNG) driver"); +MODULE_LICENSE("GPL v2"); diff --git a/drivers/char/hw_random/cctrng.c b/drivers/char/hw_random/cctrng.c index 4c50efc46483..4db198849695 100644 --- a/drivers/char/hw_random/cctrng.c +++ b/drivers/char/hw_random/cctrng.c @@ -653,7 +653,7 @@ static struct platform_driver cctrng_driver = { .pm = &cctrng_pm, }, .probe = cctrng_probe, - .remove_new = cctrng_remove, + .remove = cctrng_remove, }; module_platform_driver(cctrng_driver); diff --git a/drivers/char/hw_random/core.c b/drivers/char/hw_random/core.c index 57c51efa5613..018316f54621 100644 --- a/drivers/char/hw_random/core.c +++ b/drivers/char/hw_random/core.c @@ -181,8 +181,15 @@ static inline int rng_get_data(struct hwrng *rng, u8 *buffer, size_t size, int present; BUG_ON(!mutex_is_locked(&reading_mutex)); - if (rng->read) - return rng->read(rng, (void *)buffer, size, wait); + if (rng->read) { + int err; + + err = rng->read(rng, buffer, size, wait); + if (WARN_ON_ONCE(err > 0 && err > size)) + err = size; + + return err; + } if (rng->data_present) present = rng->data_present(rng, wait); diff --git a/drivers/char/hw_random/exynos-trng.c b/drivers/char/hw_random/exynos-trng.c index 9f039fddaee3..02e207c09e81 100644 --- a/drivers/char/hw_random/exynos-trng.c +++ b/drivers/char/hw_random/exynos-trng.c @@ -335,7 +335,7 @@ static struct platform_driver exynos_trng_driver = { .of_match_table = exynos_trng_dt_match, }, .probe = exynos_trng_probe, - .remove_new = exynos_trng_remove, + .remove = exynos_trng_remove, }; module_platform_driver(exynos_trng_driver); diff --git a/drivers/char/hw_random/histb-rng.c b/drivers/char/hw_random/histb-rng.c index f652e1135e4b..1b91e88cc4c0 100644 --- a/drivers/char/hw_random/histb-rng.c +++ b/drivers/char/hw_random/histb-rng.c @@ -89,7 +89,7 @@ depth_show(struct device *dev, struct device_attribute *attr, char *buf) struct histb_rng_priv *priv = dev_get_drvdata(dev); void __iomem *base = priv->base; - return sprintf(buf, "%d\n", histb_rng_get_depth(base)); + return sprintf(buf, "%u\n", histb_rng_get_depth(base)); } static ssize_t diff --git a/drivers/char/hw_random/ingenic-rng.c b/drivers/char/hw_random/ingenic-rng.c index 2f9b6483c4a1..bbfd662d25a6 100644 --- a/drivers/char/hw_random/ingenic-rng.c +++ b/drivers/char/hw_random/ingenic-rng.c @@ -132,7 +132,7 @@ MODULE_DEVICE_TABLE(of, ingenic_rng_of_match); static struct platform_driver ingenic_rng_driver = { .probe = ingenic_rng_probe, - .remove_new = ingenic_rng_remove, + .remove = ingenic_rng_remove, .driver = { .name = "ingenic-rng", .of_match_table = ingenic_rng_of_match, diff --git a/drivers/char/hw_random/ks-sa-rng.c b/drivers/char/hw_random/ks-sa-rng.c index 36c34252b4f6..d8fd8a354482 100644 --- a/drivers/char/hw_random/ks-sa-rng.c +++ b/drivers/char/hw_random/ks-sa-rng.c @@ -261,7 +261,7 @@ static struct platform_driver ks_sa_rng_driver = { .of_match_table = ks_sa_rng_dt_match, }, .probe = ks_sa_rng_probe, - .remove_new = ks_sa_rng_remove, + .remove = ks_sa_rng_remove, }; module_platform_driver(ks_sa_rng_driver); diff --git a/drivers/char/hw_random/mxc-rnga.c b/drivers/char/hw_random/mxc-rnga.c index f01eb95bee31..e3fcb8bcc29b 100644 --- a/drivers/char/hw_random/mxc-rnga.c +++ b/drivers/char/hw_random/mxc-rnga.c @@ -188,7 +188,7 @@ static struct platform_driver mxc_rnga_driver = { .of_match_table = mxc_rnga_of_match, }, .probe = mxc_rnga_probe, - .remove_new = mxc_rnga_remove, + .remove = mxc_rnga_remove, }; module_platform_driver(mxc_rnga_driver); diff --git a/drivers/char/hw_random/n2-drv.c b/drivers/char/hw_random/n2-drv.c index 1b49e3a86d57..ea6d5599242f 100644 --- a/drivers/char/hw_random/n2-drv.c +++ b/drivers/char/hw_random/n2-drv.c @@ -858,7 +858,7 @@ static struct platform_driver n2rng_driver = { .of_match_table = n2rng_match, }, .probe = n2rng_probe, - .remove_new = n2rng_remove, + .remove = n2rng_remove, }; module_platform_driver(n2rng_driver); diff --git a/drivers/char/hw_random/npcm-rng.c b/drivers/char/hw_random/npcm-rng.c index bce8c4829a1f..9ff00f096f38 100644 --- a/drivers/char/hw_random/npcm-rng.c +++ b/drivers/char/hw_random/npcm-rng.c @@ -176,7 +176,7 @@ static struct platform_driver npcm_rng_driver = { .of_match_table = of_match_ptr(rng_dt_id), }, .probe = npcm_rng_probe, - .remove_new = npcm_rng_remove, + .remove = npcm_rng_remove, }; module_platform_driver(npcm_rng_driver); diff --git a/drivers/char/hw_random/omap-rng.c b/drivers/char/hw_random/omap-rng.c index 4914a8720e58..5e8b50f15db7 100644 --- a/drivers/char/hw_random/omap-rng.c +++ b/drivers/char/hw_random/omap-rng.c @@ -558,7 +558,7 @@ static struct platform_driver omap_rng_driver = { .of_match_table = of_match_ptr(omap_rng_of_match), }, .probe = omap_rng_probe, - .remove_new = omap_rng_remove, + .remove = omap_rng_remove, }; module_platform_driver(omap_rng_driver); diff --git a/drivers/char/hw_random/stm32-rng.c b/drivers/char/hw_random/stm32-rng.c index 9d041a67c295..98edbe796bc5 100644 --- a/drivers/char/hw_random/stm32-rng.c +++ b/drivers/char/hw_random/stm32-rng.c @@ -4,6 +4,7 @@ */ #include <linux/clk.h> +#include <linux/clk-provider.h> #include <linux/delay.h> #include <linux/hw_random.h> #include <linux/io.h> @@ -49,6 +50,7 @@ struct stm32_rng_data { uint max_clock_rate; + uint nb_clock; u32 cr; u32 nscr; u32 htcr; @@ -72,7 +74,7 @@ struct stm32_rng_private { struct hwrng rng; struct device *dev; void __iomem *base; - struct clk *clk; + struct clk_bulk_data *clk_bulk; struct reset_control *rst; struct stm32_rng_config pm_conf; const struct stm32_rng_data *data; @@ -266,7 +268,7 @@ static uint stm32_rng_clock_freq_restrain(struct hwrng *rng) unsigned long clock_rate = 0; uint clock_div = 0; - clock_rate = clk_get_rate(priv->clk); + clock_rate = clk_get_rate(priv->clk_bulk[0].clk); /* * Get the exponent to apply on the CLKDIV field in RNG_CR register @@ -276,7 +278,7 @@ static uint stm32_rng_clock_freq_restrain(struct hwrng *rng) while ((clock_rate >> clock_div) > priv->data->max_clock_rate) clock_div++; - pr_debug("RNG clk rate : %lu\n", clk_get_rate(priv->clk) >> clock_div); + pr_debug("RNG clk rate : %lu\n", clk_get_rate(priv->clk_bulk[0].clk) >> clock_div); return clock_div; } @@ -288,7 +290,7 @@ static int stm32_rng_init(struct hwrng *rng) int err; u32 reg; - err = clk_prepare_enable(priv->clk); + err = clk_bulk_prepare_enable(priv->data->nb_clock, priv->clk_bulk); if (err) return err; @@ -328,7 +330,7 @@ static int stm32_rng_init(struct hwrng *rng) (!(reg & RNG_CR_CONDRST)), 10, 50000); if (err) { - clk_disable_unprepare(priv->clk); + clk_bulk_disable_unprepare(priv->data->nb_clock, priv->clk_bulk); dev_err(priv->dev, "%s: timeout %x!\n", __func__, reg); return -EINVAL; } @@ -356,12 +358,13 @@ static int stm32_rng_init(struct hwrng *rng) reg & RNG_SR_DRDY, 10, 100000); if (err || (reg & ~RNG_SR_DRDY)) { - clk_disable_unprepare(priv->clk); + clk_bulk_disable_unprepare(priv->data->nb_clock, priv->clk_bulk); dev_err(priv->dev, "%s: timeout:%x SR: %x!\n", __func__, err, reg); + return -EINVAL; } - clk_disable_unprepare(priv->clk); + clk_bulk_disable_unprepare(priv->data->nb_clock, priv->clk_bulk); return 0; } @@ -379,7 +382,8 @@ static int __maybe_unused stm32_rng_runtime_suspend(struct device *dev) reg = readl_relaxed(priv->base + RNG_CR); reg &= ~RNG_CR_RNGEN; writel_relaxed(reg, priv->base + RNG_CR); - clk_disable_unprepare(priv->clk); + + clk_bulk_disable_unprepare(priv->data->nb_clock, priv->clk_bulk); return 0; } @@ -389,7 +393,7 @@ static int __maybe_unused stm32_rng_suspend(struct device *dev) struct stm32_rng_private *priv = dev_get_drvdata(dev); int err; - err = clk_prepare_enable(priv->clk); + err = clk_bulk_prepare_enable(priv->data->nb_clock, priv->clk_bulk); if (err) return err; @@ -403,7 +407,7 @@ static int __maybe_unused stm32_rng_suspend(struct device *dev) writel_relaxed(priv->pm_conf.cr, priv->base + RNG_CR); - clk_disable_unprepare(priv->clk); + clk_bulk_disable_unprepare(priv->data->nb_clock, priv->clk_bulk); return 0; } @@ -414,7 +418,7 @@ static int __maybe_unused stm32_rng_runtime_resume(struct device *dev) int err; u32 reg; - err = clk_prepare_enable(priv->clk); + err = clk_bulk_prepare_enable(priv->data->nb_clock, priv->clk_bulk); if (err) return err; @@ -434,7 +438,7 @@ static int __maybe_unused stm32_rng_resume(struct device *dev) int err; u32 reg; - err = clk_prepare_enable(priv->clk); + err = clk_bulk_prepare_enable(priv->data->nb_clock, priv->clk_bulk); if (err) return err; @@ -462,7 +466,7 @@ static int __maybe_unused stm32_rng_resume(struct device *dev) reg & ~RNG_CR_CONDRST, 10, 100000); if (err) { - clk_disable_unprepare(priv->clk); + clk_bulk_disable_unprepare(priv->data->nb_clock, priv->clk_bulk); dev_err(priv->dev, "%s: timeout:%x CR: %x!\n", __func__, err, reg); return -EINVAL; } @@ -472,7 +476,7 @@ static int __maybe_unused stm32_rng_resume(struct device *dev) writel_relaxed(reg, priv->base + RNG_CR); } - clk_disable_unprepare(priv->clk); + clk_bulk_disable_unprepare(priv->data->nb_clock, priv->clk_bulk); return 0; } @@ -484,9 +488,19 @@ static const struct dev_pm_ops __maybe_unused stm32_rng_pm_ops = { stm32_rng_resume) }; +static const struct stm32_rng_data stm32mp25_rng_data = { + .has_cond_reset = true, + .max_clock_rate = 48000000, + .nb_clock = 2, + .cr = 0x00F00D00, + .nscr = 0x2B5BB, + .htcr = 0x969D, +}; + static const struct stm32_rng_data stm32mp13_rng_data = { .has_cond_reset = true, .max_clock_rate = 48000000, + .nb_clock = 1, .cr = 0x00F00D00, .nscr = 0x2B5BB, .htcr = 0x969D, @@ -494,11 +508,16 @@ static const struct stm32_rng_data stm32mp13_rng_data = { static const struct stm32_rng_data stm32_rng_data = { .has_cond_reset = false, - .max_clock_rate = 3000000, + .max_clock_rate = 48000000, + .nb_clock = 1, }; static const struct of_device_id stm32_rng_match[] = { { + .compatible = "st,stm32mp25-rng", + .data = &stm32mp25_rng_data, + }, + { .compatible = "st,stm32mp13-rng", .data = &stm32mp13_rng_data, }, @@ -516,6 +535,7 @@ static int stm32_rng_probe(struct platform_device *ofdev) struct device_node *np = ofdev->dev.of_node; struct stm32_rng_private *priv; struct resource *res; + int ret; priv = devm_kzalloc(dev, sizeof(*priv), GFP_KERNEL); if (!priv) @@ -525,10 +545,6 @@ static int stm32_rng_probe(struct platform_device *ofdev) if (IS_ERR(priv->base)) return PTR_ERR(priv->base); - priv->clk = devm_clk_get(&ofdev->dev, NULL); - if (IS_ERR(priv->clk)) - return PTR_ERR(priv->clk); - priv->rst = devm_reset_control_get(&ofdev->dev, NULL); if (!IS_ERR(priv->rst)) { reset_control_assert(priv->rst); @@ -551,6 +567,28 @@ static int stm32_rng_probe(struct platform_device *ofdev) priv->rng.read = stm32_rng_read; priv->rng.quality = 900; + if (!priv->data->nb_clock || priv->data->nb_clock > 2) + return -EINVAL; + + ret = devm_clk_bulk_get_all(dev, &priv->clk_bulk); + if (ret != priv->data->nb_clock) + return dev_err_probe(dev, -EINVAL, "Failed to get clocks: %d\n", ret); + + if (priv->data->nb_clock == 2) { + const char *id = priv->clk_bulk[1].id; + struct clk *clk = priv->clk_bulk[1].clk; + + if (!priv->clk_bulk[0].id || !priv->clk_bulk[1].id) + return dev_err_probe(dev, -EINVAL, "Missing clock name\n"); + + if (strcmp(priv->clk_bulk[0].id, "core")) { + priv->clk_bulk[1].id = priv->clk_bulk[0].id; + priv->clk_bulk[1].clk = priv->clk_bulk[0].clk; + priv->clk_bulk[0].id = id; + priv->clk_bulk[0].clk = clk; + } + } + pm_runtime_set_autosuspend_delay(dev, 100); pm_runtime_use_autosuspend(dev); pm_runtime_enable(dev); @@ -565,7 +603,7 @@ static struct platform_driver stm32_rng_driver = { .of_match_table = stm32_rng_match, }, .probe = stm32_rng_probe, - .remove_new = stm32_rng_remove, + .remove = stm32_rng_remove, }; module_platform_driver(stm32_rng_driver); diff --git a/drivers/char/hw_random/timeriomem-rng.c b/drivers/char/hw_random/timeriomem-rng.c index 65b8260339f5..7174bfccc7b3 100644 --- a/drivers/char/hw_random/timeriomem-rng.c +++ b/drivers/char/hw_random/timeriomem-rng.c @@ -193,7 +193,7 @@ static struct platform_driver timeriomem_rng_driver = { .of_match_table = timeriomem_rng_match, }, .probe = timeriomem_rng_probe, - .remove_new = timeriomem_rng_remove, + .remove = timeriomem_rng_remove, }; module_platform_driver(timeriomem_rng_driver); diff --git a/drivers/char/hw_random/xgene-rng.c b/drivers/char/hw_random/xgene-rng.c index 642d13519464..39acaa503fec 100644 --- a/drivers/char/hw_random/xgene-rng.c +++ b/drivers/char/hw_random/xgene-rng.c @@ -375,7 +375,7 @@ MODULE_DEVICE_TABLE(of, xgene_rng_of_match); static struct platform_driver xgene_rng_driver = { .probe = xgene_rng_probe, - .remove_new = xgene_rng_remove, + .remove = xgene_rng_remove, .driver = { .name = "xgene-rng", .of_match_table = xgene_rng_of_match, diff --git a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c index 890664bd5f0f..58a76e2ba64e 100644 --- a/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c +++ b/drivers/crypto/allwinner/sun4i-ss/sun4i-ss-core.c @@ -542,7 +542,7 @@ MODULE_DEVICE_TABLE(of, a20ss_crypto_of_match_table); static struct platform_driver sun4i_ss_driver = { .probe = sun4i_ss_probe, - .remove_new = sun4i_ss_remove, + .remove = sun4i_ss_remove, .driver = { .name = "sun4i-ss", .pm = &sun4i_ss_pm_ops, diff --git a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c index e55e58e164db..ec1ffda9ea32 100644 --- a/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c +++ b/drivers/crypto/allwinner/sun8i-ce/sun8i-ce-core.c @@ -1129,7 +1129,7 @@ MODULE_DEVICE_TABLE(of, sun8i_ce_crypto_of_match_table); static struct platform_driver sun8i_ce_driver = { .probe = sun8i_ce_probe, - .remove_new = sun8i_ce_remove, + .remove = sun8i_ce_remove, .driver = { .name = "sun8i-ce", .pm = &sun8i_ce_pm_ops, diff --git a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c index 0dbc0220146c..f45685707e0d 100644 --- a/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c +++ b/drivers/crypto/allwinner/sun8i-ss/sun8i-ss-core.c @@ -929,7 +929,7 @@ MODULE_DEVICE_TABLE(of, sun8i_ss_crypto_of_match_table); static struct platform_driver sun8i_ss_driver = { .probe = sun8i_ss_probe, - .remove_new = sun8i_ss_remove, + .remove = sun8i_ss_remove, .driver = { .name = "sun8i-ss", .pm = &sun8i_ss_pm_ops, diff --git a/drivers/crypto/amcc/crypto4xx_core.c b/drivers/crypto/amcc/crypto4xx_core.c index 6006703fb6d7..ec3ccfa60445 100644 --- a/drivers/crypto/amcc/crypto4xx_core.c +++ b/drivers/crypto/amcc/crypto4xx_core.c @@ -653,9 +653,6 @@ static void crypto4xx_stop_all(struct crypto4xx_core_device *core_dev) crypto4xx_destroy_pdr(core_dev->dev); crypto4xx_destroy_gdr(core_dev->dev); crypto4xx_destroy_sdr(core_dev->dev); - iounmap(core_dev->dev->ce_base); - kfree(core_dev->dev); - kfree(core_dev); } static u32 get_next_gd(u32 current) @@ -1333,17 +1330,12 @@ static struct crypto4xx_alg_common crypto4xx_alg[] = { static int crypto4xx_probe(struct platform_device *ofdev) { int rc; - struct resource res; struct device *dev = &ofdev->dev; struct crypto4xx_core_device *core_dev; struct device_node *np; u32 pvr; bool is_revb = true; - rc = of_address_to_resource(ofdev->dev.of_node, 0, &res); - if (rc) - return -ENODEV; - np = of_find_compatible_node(NULL, NULL, "amcc,ppc460ex-crypto"); if (np) { mtdcri(SDR0, PPC460EX_SDR0_SRST, @@ -1374,16 +1366,17 @@ static int crypto4xx_probe(struct platform_device *ofdev) of_node_put(np); - core_dev = kzalloc(sizeof(struct crypto4xx_core_device), GFP_KERNEL); + core_dev = devm_kzalloc( + &ofdev->dev, sizeof(struct crypto4xx_core_device), GFP_KERNEL); if (!core_dev) return -ENOMEM; dev_set_drvdata(dev, core_dev); core_dev->ofdev = ofdev; - core_dev->dev = kzalloc(sizeof(struct crypto4xx_device), GFP_KERNEL); - rc = -ENOMEM; + core_dev->dev = devm_kzalloc( + &ofdev->dev, sizeof(struct crypto4xx_device), GFP_KERNEL); if (!core_dev->dev) - goto err_alloc_dev; + return -ENOMEM; /* * Older version of 460EX/GT have a hardware bug. @@ -1402,7 +1395,9 @@ static int crypto4xx_probe(struct platform_device *ofdev) core_dev->dev->core_dev = core_dev; core_dev->dev->is_revb = is_revb; core_dev->device = dev; - mutex_init(&core_dev->rng_lock); + rc = devm_mutex_init(&ofdev->dev, &core_dev->rng_lock); + if (rc) + return rc; spin_lock_init(&core_dev->lock); INIT_LIST_HEAD(&core_dev->dev->alg_list); ratelimit_default_init(&core_dev->dev->aead_ratelimit); @@ -1421,21 +1416,21 @@ static int crypto4xx_probe(struct platform_device *ofdev) tasklet_init(&core_dev->tasklet, crypto4xx_bh_tasklet_cb, (unsigned long) dev); - core_dev->dev->ce_base = of_iomap(ofdev->dev.of_node, 0); - if (!core_dev->dev->ce_base) { - dev_err(dev, "failed to of_iomap\n"); - rc = -ENOMEM; - goto err_iomap; + core_dev->dev->ce_base = devm_platform_ioremap_resource(ofdev, 0); + if (IS_ERR(core_dev->dev->ce_base)) { + dev_err(&ofdev->dev, "failed to ioremap resource"); + rc = PTR_ERR(core_dev->dev->ce_base); + goto err_build_sdr; } /* Register for Crypto isr, Crypto Engine IRQ */ core_dev->irq = irq_of_parse_and_map(ofdev->dev.of_node, 0); - rc = request_irq(core_dev->irq, is_revb ? - crypto4xx_ce_interrupt_handler_revb : - crypto4xx_ce_interrupt_handler, 0, - KBUILD_MODNAME, dev); + rc = devm_request_irq(&ofdev->dev, core_dev->irq, + is_revb ? crypto4xx_ce_interrupt_handler_revb : + crypto4xx_ce_interrupt_handler, + 0, KBUILD_MODNAME, dev); if (rc) - goto err_request_irq; + goto err_iomap; /* need to setup pdr, rdr, gdr and sdr before this */ crypto4xx_hw_init(core_dev->dev); @@ -1444,26 +1439,17 @@ static int crypto4xx_probe(struct platform_device *ofdev) rc = crypto4xx_register_alg(core_dev->dev, crypto4xx_alg, ARRAY_SIZE(crypto4xx_alg)); if (rc) - goto err_start_dev; + goto err_iomap; ppc4xx_trng_probe(core_dev); return 0; -err_start_dev: - free_irq(core_dev->irq, dev); -err_request_irq: - irq_dispose_mapping(core_dev->irq); - iounmap(core_dev->dev->ce_base); err_iomap: tasklet_kill(&core_dev->tasklet); err_build_sdr: crypto4xx_destroy_sdr(core_dev->dev); crypto4xx_destroy_gdr(core_dev->dev); crypto4xx_destroy_pdr(core_dev->dev); - kfree(core_dev->dev); -err_alloc_dev: - kfree(core_dev); - return rc; } @@ -1474,13 +1460,9 @@ static void crypto4xx_remove(struct platform_device *ofdev) ppc4xx_trng_remove(core_dev); - free_irq(core_dev->irq, dev); - irq_dispose_mapping(core_dev->irq); - tasklet_kill(&core_dev->tasklet); /* Un-register with Linux CryptoAPI */ crypto4xx_unregister_alg(core_dev->dev); - mutex_destroy(&core_dev->rng_lock); /* Free all allocated memory */ crypto4xx_stop_all(core_dev); } @@ -1497,7 +1479,7 @@ static struct platform_driver crypto4xx_driver = { .of_match_table = crypto4xx_match, }, .probe = crypto4xx_probe, - .remove_new = crypto4xx_remove, + .remove = crypto4xx_remove, }; module_platform_driver(crypto4xx_driver); diff --git a/drivers/crypto/amlogic/amlogic-gxl-core.c b/drivers/crypto/amlogic/amlogic-gxl-core.c index f54ab0d0b1e8..1c18a5b8470e 100644 --- a/drivers/crypto/amlogic/amlogic-gxl-core.c +++ b/drivers/crypto/amlogic/amlogic-gxl-core.c @@ -240,11 +240,9 @@ static int meson_crypto_probe(struct platform_device *pdev) platform_set_drvdata(pdev, mc); mc->base = devm_platform_ioremap_resource(pdev, 0); - if (IS_ERR(mc->base)) { - err = PTR_ERR(mc->base); - dev_err(&pdev->dev, "Cannot request MMIO err=%d\n", err); - return err; - } + if (IS_ERR(mc->base)) + return PTR_ERR(mc->base); + mc->busclk = devm_clk_get(&pdev->dev, "blkmv"); if (IS_ERR(mc->busclk)) { err = PTR_ERR(mc->busclk); @@ -322,7 +320,7 @@ MODULE_DEVICE_TABLE(of, meson_crypto_of_match_table); static struct platform_driver meson_crypto_driver = { .probe = meson_crypto_probe, - .remove_new = meson_crypto_remove, + .remove = meson_crypto_remove, .driver = { .name = "gxl-crypto", .of_match_table = meson_crypto_of_match_table, diff --git a/drivers/crypto/aspeed/aspeed-acry.c b/drivers/crypto/aspeed/aspeed-acry.c index b4613bd4ad96..8d1c79aaca07 100644 --- a/drivers/crypto/aspeed/aspeed-acry.c +++ b/drivers/crypto/aspeed/aspeed-acry.c @@ -601,8 +601,6 @@ static struct aspeed_acry_alg aspeed_acry_akcipher_algs[] = { .akcipher.base = { .encrypt = aspeed_acry_rsa_enc, .decrypt = aspeed_acry_rsa_dec, - .sign = aspeed_acry_rsa_dec, - .verify = aspeed_acry_rsa_enc, .set_pub_key = aspeed_acry_rsa_set_pub_key, .set_priv_key = aspeed_acry_rsa_set_priv_key, .max_size = aspeed_acry_rsa_max_size, @@ -808,7 +806,7 @@ MODULE_DEVICE_TABLE(of, aspeed_acry_of_matches); static struct platform_driver aspeed_acry_driver = { .probe = aspeed_acry_probe, - .remove_new = aspeed_acry_remove, + .remove = aspeed_acry_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = aspeed_acry_of_matches, diff --git a/drivers/crypto/aspeed/aspeed-hace.c b/drivers/crypto/aspeed/aspeed-hace.c index 062f2a66dd23..3fe644bfe037 100644 --- a/drivers/crypto/aspeed/aspeed-hace.c +++ b/drivers/crypto/aspeed/aspeed-hace.c @@ -266,7 +266,7 @@ MODULE_DEVICE_TABLE(of, aspeed_hace_of_matches); static struct platform_driver aspeed_hace_driver = { .probe = aspeed_hace_probe, - .remove_new = aspeed_hace_remove, + .remove = aspeed_hace_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = aspeed_hace_of_matches, diff --git a/drivers/crypto/atmel-aes.c b/drivers/crypto/atmel-aes.c index 0dd90785db9a..14bf86957d31 100644 --- a/drivers/crypto/atmel-aes.c +++ b/drivers/crypto/atmel-aes.c @@ -2453,7 +2453,7 @@ static void atmel_aes_remove(struct platform_device *pdev) static struct platform_driver atmel_aes_driver = { .probe = atmel_aes_probe, - .remove_new = atmel_aes_remove, + .remove = atmel_aes_remove, .driver = { .name = "atmel_aes", .of_match_table = atmel_aes_dt_ids, diff --git a/drivers/crypto/atmel-ecc.c b/drivers/crypto/atmel-ecc.c index 590ea984c622..0d48e64d28b1 100644 --- a/drivers/crypto/atmel-ecc.c +++ b/drivers/crypto/atmel-ecc.c @@ -379,7 +379,7 @@ MODULE_DEVICE_TABLE(of, atmel_ecc_dt_ids); #endif static const struct i2c_device_id atmel_ecc_id[] = { - { "atecc508a", 0 }, + { "atecc508a" }, { } }; MODULE_DEVICE_TABLE(i2c, atmel_ecc_id); diff --git a/drivers/crypto/atmel-sha.c b/drivers/crypto/atmel-sha.c index 8cc57df25778..67a170608566 100644 --- a/drivers/crypto/atmel-sha.c +++ b/drivers/crypto/atmel-sha.c @@ -2691,7 +2691,7 @@ static void atmel_sha_remove(struct platform_device *pdev) static struct platform_driver atmel_sha_driver = { .probe = atmel_sha_probe, - .remove_new = atmel_sha_remove, + .remove = atmel_sha_remove, .driver = { .name = "atmel_sha", .of_match_table = atmel_sha_dt_ids, diff --git a/drivers/crypto/atmel-sha204a.c b/drivers/crypto/atmel-sha204a.c index a02d496f4c41..75bebec2c757 100644 --- a/drivers/crypto/atmel-sha204a.c +++ b/drivers/crypto/atmel-sha204a.c @@ -202,8 +202,8 @@ static const struct of_device_id atmel_sha204a_dt_ids[] __maybe_unused = { MODULE_DEVICE_TABLE(of, atmel_sha204a_dt_ids); static const struct i2c_device_id atmel_sha204a_id[] = { - { "atsha204", 0 }, - { "atsha204a", 0 }, + { "atsha204" }, + { "atsha204a" }, { /* sentinel */ } }; MODULE_DEVICE_TABLE(i2c, atmel_sha204a_id); diff --git a/drivers/crypto/atmel-tdes.c b/drivers/crypto/atmel-tdes.c index dcc2380a5889..de9717e221e4 100644 --- a/drivers/crypto/atmel-tdes.c +++ b/drivers/crypto/atmel-tdes.c @@ -872,7 +872,7 @@ static void atmel_tdes_done_task(unsigned long data) if (!err) err = atmel_tdes_crypt_start(dd); if (!err) - return; /* DMA started. Not fininishing. */ + return; /* DMA started. Not finishing. */ } atmel_tdes_finish_req(dd, err); @@ -1074,7 +1074,7 @@ static void atmel_tdes_remove(struct platform_device *pdev) static struct platform_driver atmel_tdes_driver = { .probe = atmel_tdes_probe, - .remove_new = atmel_tdes_remove, + .remove = atmel_tdes_remove, .driver = { .name = "atmel_tdes", .of_match_table = atmel_tdes_dt_ids, diff --git a/drivers/crypto/axis/artpec6_crypto.c b/drivers/crypto/axis/artpec6_crypto.c index 75440ea6206e..1c1f57baef0e 100644 --- a/drivers/crypto/axis/artpec6_crypto.c +++ b/drivers/crypto/axis/artpec6_crypto.c @@ -2975,7 +2975,7 @@ static void artpec6_crypto_remove(struct platform_device *pdev) static struct platform_driver artpec6_crypto_driver = { .probe = artpec6_crypto_probe, - .remove_new = artpec6_crypto_remove, + .remove = artpec6_crypto_remove, .driver = { .name = "artpec6-crypto", .of_match_table = artpec6_crypto_of_match, diff --git a/drivers/crypto/bcm/cipher.c b/drivers/crypto/bcm/cipher.c index 1a3ecd44cbaf..9e6798efbfb7 100644 --- a/drivers/crypto/bcm/cipher.c +++ b/drivers/crypto/bcm/cipher.c @@ -2415,6 +2415,7 @@ static int ahash_hmac_setkey(struct crypto_ahash *ahash, const u8 *key, static int ahash_hmac_init(struct ahash_request *req) { + int ret; struct iproc_reqctx_s *rctx = ahash_request_ctx(req); struct crypto_ahash *tfm = crypto_ahash_reqtfm(req); struct iproc_ctx_s *ctx = crypto_ahash_ctx(tfm); @@ -2424,7 +2425,9 @@ static int ahash_hmac_init(struct ahash_request *req) flow_log("ahash_hmac_init()\n"); /* init the context as a hash */ - ahash_init(req); + ret = ahash_init(req); + if (ret) + return ret; if (!spu_no_incr_hash(ctx)) { /* SPU-M can do incr hashing but needs sw for outer HMAC */ @@ -4704,7 +4707,7 @@ static struct platform_driver bcm_spu_pdriver = { .of_match_table = of_match_ptr(bcm_spu_dt_ids), }, .probe = bcm_spu_probe, - .remove_new = bcm_spu_remove, + .remove = bcm_spu_remove, }; module_platform_driver(bcm_spu_pdriver); diff --git a/drivers/crypto/caam/caampkc.c b/drivers/crypto/caam/caampkc.c index 887a5f2fb927..cb001aa1de66 100644 --- a/drivers/crypto/caam/caampkc.c +++ b/drivers/crypto/caam/caampkc.c @@ -984,7 +984,7 @@ err: return -ENOMEM; } -static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, +static int caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, struct rsa_key *raw_key) { struct caam_rsa_key *rsa_key = &ctx->key; @@ -994,7 +994,7 @@ static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, rsa_key->p = caam_read_raw_data(raw_key->p, &p_sz); if (!rsa_key->p) - return; + return -ENOMEM; rsa_key->p_sz = p_sz; rsa_key->q = caam_read_raw_data(raw_key->q, &q_sz); @@ -1029,7 +1029,7 @@ static void caam_rsa_set_priv_key_form(struct caam_rsa_ctx *ctx, rsa_key->priv_form = FORM3; - return; + return 0; free_dq: kfree_sensitive(rsa_key->dq); @@ -1043,6 +1043,7 @@ free_q: kfree_sensitive(rsa_key->q); free_p: kfree_sensitive(rsa_key->p); + return -ENOMEM; } static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, @@ -1088,7 +1089,9 @@ static int caam_rsa_set_priv_key(struct crypto_akcipher *tfm, const void *key, rsa_key->e_sz = raw_key.e_sz; rsa_key->n_sz = raw_key.n_sz; - caam_rsa_set_priv_key_form(ctx, &raw_key); + ret = caam_rsa_set_priv_key_form(ctx, &raw_key); + if (ret) + goto err; return 0; diff --git a/drivers/crypto/caam/jr.c b/drivers/crypto/caam/jr.c index 26eba7de3fb0..9fcdb64084ac 100644 --- a/drivers/crypto/caam/jr.c +++ b/drivers/crypto/caam/jr.c @@ -819,7 +819,7 @@ static struct platform_driver caam_jr_driver = { .pm = pm_ptr(&caam_jr_pm_ops), }, .probe = caam_jr_probe, - .remove_new = caam_jr_remove, + .remove = caam_jr_remove, .shutdown = caam_jr_remove, }; diff --git a/drivers/crypto/caam/qi.c b/drivers/crypto/caam/qi.c index f6111ee9ed34..7701d00bcb3a 100644 --- a/drivers/crypto/caam/qi.c +++ b/drivers/crypto/caam/qi.c @@ -733,7 +733,7 @@ static void free_caam_qi_pcpu_netdev(const cpumask_t *cpus) int caam_qi_init(struct platform_device *caam_pdev) { int err, i; - struct device *ctrldev = &caam_pdev->dev, *qidev; + struct device *qidev = &caam_pdev->dev; struct caam_drv_private *ctrlpriv; const cpumask_t *cpus = qman_affine_cpus(); cpumask_var_t clean_mask; @@ -742,8 +742,7 @@ int caam_qi_init(struct platform_device *caam_pdev) if (!zalloc_cpumask_var(&clean_mask, GFP_KERNEL)) goto fail_cpumask; - ctrlpriv = dev_get_drvdata(ctrldev); - qidev = ctrldev; + ctrlpriv = dev_get_drvdata(qidev); /* Initialize the congestion detection */ err = init_cgr(qidev); @@ -794,7 +793,7 @@ int caam_qi_init(struct platform_device *caam_pdev) caam_debugfs_qi_init(ctrlpriv); - err = devm_add_action_or_reset(qidev, caam_qi_shutdown, ctrlpriv); + err = devm_add_action_or_reset(qidev, caam_qi_shutdown, qidev); if (err) goto fail2; diff --git a/drivers/crypto/cavium/cpt/cptpf_main.c b/drivers/crypto/cavium/cpt/cptpf_main.c index 6872ac344001..54de869e5374 100644 --- a/drivers/crypto/cavium/cpt/cptpf_main.c +++ b/drivers/crypto/cavium/cpt/cptpf_main.c @@ -44,7 +44,7 @@ static void cpt_disable_cores(struct cpt_device *cpt, u64 coremask, dev_err(dev, "Cores still busy %llx", coremask); grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0)); - if (timeout--) + if (!timeout--) break; udelay(CSR_DELAY); @@ -302,6 +302,8 @@ static int cpt_ucode_load_fw(struct cpt_device *cpt, const u8 *fw, bool is_ae) ret = do_cpt_init(cpt, mcode); if (ret) { + dma_free_coherent(&cpt->pdev->dev, mcode->code_size, + mcode->code, mcode->phys_base); dev_err(dev, "do_cpt_init failed with ret: %d\n", ret); goto fw_release; } @@ -394,7 +396,7 @@ static void cpt_disable_all_cores(struct cpt_device *cpt) dev_err(dev, "Cores still busy"); grp = cpt_read_csr64(cpt->reg_base, CPTX_PF_EXEC_BUSY(0)); - if (timeout--) + if (!timeout--) break; udelay(CSR_DELAY); diff --git a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c index 153004bdfb5c..fb59bb282455 100644 --- a/drivers/crypto/cavium/cpt/cptvf_reqmanager.c +++ b/drivers/crypto/cavium/cpt/cptvf_reqmanager.c @@ -238,7 +238,7 @@ static int send_cpt_command(struct cpt_vf *cptvf, union cpt_inst_s *cmd, qinfo = &cptvf->cqinfo; queue = &qinfo->queue[qno]; - /* lock commad queue */ + /* lock command queue */ spin_lock(&queue->lock); ent = &queue->qhead->head[queue->idx * qinfo->cmd_size]; memcpy(ent, (void *)cmd, qinfo->cmd_size); @@ -510,7 +510,7 @@ get_pending_entry: info->time_in = jiffies; info->req = req; - /* Create the CPT_INST_S type command for HW intrepretation */ + /* Create the CPT_INST_S type command for HW interpretation */ cptinst.s.doneint = true; cptinst.s.res_addr = (u64)info->comp_baddr; cptinst.s.tag = 0; diff --git a/drivers/crypto/cavium/nitrox/nitrox_lib.c b/drivers/crypto/cavium/nitrox/nitrox_lib.c index a5cdc2b48bd6..068265207ddd 100644 --- a/drivers/crypto/cavium/nitrox/nitrox_lib.c +++ b/drivers/crypto/cavium/nitrox/nitrox_lib.c @@ -17,7 +17,7 @@ #define CRYPTO_CTX_SIZE 256 -/* packet inuput ring alignments */ +/* packet input ring alignments */ #define PKTIN_Q_ALIGN_BYTES 16 /* AQM Queue input alignments */ #define AQM_Q_ALIGN_BYTES 32 diff --git a/drivers/crypto/ccp/sp-platform.c b/drivers/crypto/ccp/sp-platform.c index ff6ceb4feee0..3933cac1694d 100644 --- a/drivers/crypto/ccp/sp-platform.c +++ b/drivers/crypto/ccp/sp-platform.c @@ -210,7 +210,7 @@ static struct platform_driver sp_platform_driver = { .of_match_table = sp_of_match, }, .probe = sp_platform_probe, - .remove_new = sp_platform_remove, + .remove = sp_platform_remove, #ifdef CONFIG_PM .suspend = sp_platform_suspend, .resume = sp_platform_resume, diff --git a/drivers/crypto/ccree/cc_aead.c b/drivers/crypto/ccree/cc_aead.c index 5ef39d682389..81533681f7fb 100644 --- a/drivers/crypto/ccree/cc_aead.c +++ b/drivers/crypto/ccree/cc_aead.c @@ -2226,7 +2226,7 @@ static int cc_rfc4543_gcm_encrypt(struct aead_request *req) memset(areq_ctx, 0, sizeof(*areq_ctx)); - //plaintext is not encryped with rfc4543 + //plaintext is not encrypted with rfc4543 areq_ctx->plaintext_authenticate_only = true; /* No generated IV required */ @@ -2277,7 +2277,7 @@ static int cc_rfc4543_gcm_decrypt(struct aead_request *req) memset(areq_ctx, 0, sizeof(*areq_ctx)); - //plaintext is not decryped with rfc4543 + //plaintext is not decrypted with rfc4543 areq_ctx->plaintext_authenticate_only = true; /* No generated IV required */ diff --git a/drivers/crypto/ccree/cc_cipher.c b/drivers/crypto/ccree/cc_cipher.c index 3fb667a17bbb..d39c067672fd 100644 --- a/drivers/crypto/ccree/cc_cipher.c +++ b/drivers/crypto/ccree/cc_cipher.c @@ -179,7 +179,7 @@ static int cc_cipher_init(struct crypto_tfm *tfm) } max_key_buf_size <<= 1; - /* Alloc fallabck tfm or essiv when key size != 256 bit */ + /* Alloc fallback tfm or essiv when key size != 256 bit */ ctx_p->fallback_tfm = crypto_alloc_skcipher(name, 0, CRYPTO_ALG_NEED_FALLBACK | CRYPTO_ALG_ASYNC); diff --git a/drivers/crypto/ccree/cc_driver.c b/drivers/crypto/ccree/cc_driver.c index 9177b54bb0f5..061e68a31c36 100644 --- a/drivers/crypto/ccree/cc_driver.c +++ b/drivers/crypto/ccree/cc_driver.c @@ -643,7 +643,7 @@ static struct platform_driver ccree_driver = { #endif }, .probe = ccree_probe, - .remove_new = ccree_remove, + .remove = ccree_remove, }; static int __init ccree_init(void) diff --git a/drivers/crypto/ccree/cc_hash.c b/drivers/crypto/ccree/cc_hash.c index f418162932fe..d0612bec4d58 100644 --- a/drivers/crypto/ccree/cc_hash.c +++ b/drivers/crypto/ccree/cc_hash.c @@ -1577,7 +1577,7 @@ struct cc_hash_template { /* hash descriptors */ static struct cc_hash_template driver_hash[] = { - //Asynchronize hash template + //Asynchronous hash template { .name = "sha1", .driver_name = "sha1-ccree", diff --git a/drivers/crypto/chelsio/chcr_algo.c b/drivers/crypto/chelsio/chcr_algo.c index 177428480c7d..af37477ffd8d 100644 --- a/drivers/crypto/chelsio/chcr_algo.c +++ b/drivers/crypto/chelsio/chcr_algo.c @@ -1186,7 +1186,7 @@ static int chcr_handle_cipher_resp(struct skcipher_request *req, else bytes = rounddown(bytes, 16); } else { - /*CTR mode counter overfloa*/ + /*CTR mode counter overflow*/ bytes = req->cryptlen - reqctx->processed; } err = chcr_update_cipher_iv(req, fw6_pld, reqctx->iv); diff --git a/drivers/crypto/exynos-rng.c b/drivers/crypto/exynos-rng.c index 0dd8baf16cb4..2aaa98f9b44e 100644 --- a/drivers/crypto/exynos-rng.c +++ b/drivers/crypto/exynos-rng.c @@ -389,7 +389,7 @@ static struct platform_driver exynos_rng_driver = { .of_match_table = exynos_rng_dt_match, }, .probe = exynos_rng_probe, - .remove_new = exynos_rng_remove, + .remove = exynos_rng_remove, }; module_platform_driver(exynos_rng_driver); diff --git a/drivers/crypto/gemini/sl3516-ce-core.c b/drivers/crypto/gemini/sl3516-ce-core.c index 1d1a889599bb..f7e0e3fea15c 100644 --- a/drivers/crypto/gemini/sl3516-ce-core.c +++ b/drivers/crypto/gemini/sl3516-ce-core.c @@ -528,7 +528,7 @@ MODULE_DEVICE_TABLE(of, sl3516_ce_crypto_of_match_table); static struct platform_driver sl3516_ce_driver = { .probe = sl3516_ce_probe, - .remove_new = sl3516_ce_remove, + .remove = sl3516_ce_remove, .driver = { .name = "sl3516-crypto", .pm = &sl3516_ce_pm_ops, diff --git a/drivers/crypto/hisilicon/hpre/hpre.h b/drivers/crypto/hisilicon/hpre/hpre.h index 9f0b94c8e03d..0f3ddbadbcf9 100644 --- a/drivers/crypto/hisilicon/hpre/hpre.h +++ b/drivers/crypto/hisilicon/hpre/hpre.h @@ -100,6 +100,29 @@ struct hpre_sqe { __le32 rsvd1[_HPRE_SQE_ALIGN_EXT]; }; +enum hpre_cap_table_type { + QM_RAS_NFE_TYPE = 0x0, + QM_RAS_NFE_RESET, + QM_RAS_CE_TYPE, + HPRE_RAS_NFE_TYPE, + HPRE_RAS_NFE_RESET, + HPRE_RAS_CE_TYPE, + HPRE_CORE_INFO, + HPRE_CORE_EN, + HPRE_DRV_ALG_BITMAP, + HPRE_ALG_BITMAP, + HPRE_CORE1_BITMAP_CAP, + HPRE_CORE2_BITMAP_CAP, + HPRE_CORE3_BITMAP_CAP, + HPRE_CORE4_BITMAP_CAP, + HPRE_CORE5_BITMAP_CAP, + HPRE_CORE6_BITMAP_CAP, + HPRE_CORE7_BITMAP_CAP, + HPRE_CORE8_BITMAP_CAP, + HPRE_CORE9_BITMAP_CAP, + HPRE_CORE10_BITMAP_CAP, +}; + struct hisi_qp *hpre_create_qp(u8 type); int hpre_algs_register(struct hisi_qm *qm); void hpre_algs_unregister(struct hisi_qm *qm); diff --git a/drivers/crypto/hisilicon/hpre/hpre_crypto.c b/drivers/crypto/hisilicon/hpre/hpre_crypto.c index c167dbd6c7d6..2a2910261210 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_crypto.c +++ b/drivers/crypto/hisilicon/hpre/hpre_crypto.c @@ -2006,8 +2006,6 @@ static void hpre_curve25519_exit_tfm(struct crypto_kpp *tfm) } static struct akcipher_alg rsa = { - .sign = hpre_rsa_dec, - .verify = hpre_rsa_enc, .encrypt = hpre_rsa_enc, .decrypt = hpre_rsa_dec, .set_pub_key = hpre_rsa_setpubkey, diff --git a/drivers/crypto/hisilicon/hpre/hpre_main.c b/drivers/crypto/hisilicon/hpre/hpre_main.c index 6b536ad2ada5..96fde9437b4b 100644 --- a/drivers/crypto/hisilicon/hpre/hpre_main.c +++ b/drivers/crypto/hisilicon/hpre/hpre_main.c @@ -13,6 +13,7 @@ #include <linux/uacce.h> #include "hpre.h" +#define CAP_FILE_PERMISSION 0444 #define HPRE_CTRL_CNT_CLR_CE_BIT BIT(0) #define HPRE_CTRL_CNT_CLR_CE 0x301000 #define HPRE_FSM_MAX_CNT 0x301008 @@ -203,7 +204,7 @@ static const struct hisi_qm_cap_info hpre_basic_info[] = { {HPRE_RESET_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x3FFFFE, 0xBFFC3E}, {HPRE_OOO_SHUTDOWN_MASK_CAP, 0x3134, 0, GENMASK(31, 0), 0x0, 0x22, 0xBFFC3E}, {HPRE_CE_MASK_CAP, 0x3138, 0, GENMASK(31, 0), 0x0, 0x1, 0x1}, - {HPRE_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x0, 0x4, 0x1}, + {HPRE_CLUSTER_NUM_CAP, 0x313c, 20, GENMASK(3, 0), 0x0, 0x4, 0x1}, {HPRE_CORE_TYPE_NUM_CAP, 0x313c, 16, GENMASK(3, 0), 0x0, 0x2, 0x2}, {HPRE_CORE_NUM_CAP, 0x313c, 8, GENMASK(7, 0), 0x0, 0x8, 0xA}, {HPRE_CLUSTER_CORE_NUM_CAP, 0x313c, 0, GENMASK(7, 0), 0x0, 0x2, 0xA}, @@ -222,18 +223,27 @@ static const struct hisi_qm_cap_info hpre_basic_info[] = { {HPRE_CORE10_ALG_BITMAP_CAP, 0x3170, 0, GENMASK(31, 0), 0x0, 0x10, 0x10} }; -enum hpre_pre_store_cap_idx { - HPRE_CLUSTER_NUM_CAP_IDX = 0x0, - HPRE_CORE_ENABLE_BITMAP_CAP_IDX, - HPRE_DRV_ALG_BITMAP_CAP_IDX, - HPRE_DEV_ALG_BITMAP_CAP_IDX, -}; - -static const u32 hpre_pre_store_caps[] = { - HPRE_CLUSTER_NUM_CAP, - HPRE_CORE_ENABLE_BITMAP_CAP, - HPRE_DRV_ALG_BITMAP_CAP, - HPRE_DEV_ALG_BITMAP_CAP, +static const struct hisi_qm_cap_query_info hpre_cap_query_info[] = { + {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C37, 0x7C37}, + {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC77, 0x6C77}, + {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, + {HPRE_RAS_NFE_TYPE, "HPRE_RAS_NFE_TYPE ", 0x3130, 0x0, 0x3FFFFE, 0x1FFFC3E}, + {HPRE_RAS_NFE_RESET, "HPRE_RAS_NFE_RESET ", 0x3134, 0x0, 0x3FFFFE, 0xBFFC3E}, + {HPRE_RAS_CE_TYPE, "HPRE_RAS_CE_TYPE ", 0x3138, 0x0, 0x1, 0x1}, + {HPRE_CORE_INFO, "HPRE_CORE_INFO ", 0x313c, 0x0, 0x420802, 0x120A0A}, + {HPRE_CORE_EN, "HPRE_CORE_EN ", 0x3140, 0x0, 0xF, 0x3FF}, + {HPRE_DRV_ALG_BITMAP, "HPRE_DRV_ALG_BITMAP ", 0x3144, 0x0, 0x03, 0x27}, + {HPRE_ALG_BITMAP, "HPRE_ALG_BITMAP ", 0x3148, 0x0, 0x03, 0x7F}, + {HPRE_CORE1_BITMAP_CAP, "HPRE_CORE1_BITMAP_CAP ", 0x314c, 0x0, 0x7F, 0x7F}, + {HPRE_CORE2_BITMAP_CAP, "HPRE_CORE2_BITMAP_CAP ", 0x3150, 0x0, 0x7F, 0x7F}, + {HPRE_CORE3_BITMAP_CAP, "HPRE_CORE3_BITMAP_CAP ", 0x3154, 0x0, 0x7F, 0x7F}, + {HPRE_CORE4_BITMAP_CAP, "HPRE_CORE4_BITMAP_CAP ", 0x3158, 0x0, 0x7F, 0x7F}, + {HPRE_CORE5_BITMAP_CAP, "HPRE_CORE5_BITMAP_CAP ", 0x315c, 0x0, 0x7F, 0x7F}, + {HPRE_CORE6_BITMAP_CAP, "HPRE_CORE6_BITMAP_CAP ", 0x3160, 0x0, 0x7F, 0x7F}, + {HPRE_CORE7_BITMAP_CAP, "HPRE_CORE7_BITMAP_CAP ", 0x3164, 0x0, 0x7F, 0x7F}, + {HPRE_CORE8_BITMAP_CAP, "HPRE_CORE8_BITMAP_CAP ", 0x3168, 0x0, 0x7F, 0x7F}, + {HPRE_CORE9_BITMAP_CAP, "HPRE_CORE9_BITMAP_CAP ", 0x316c, 0x0, 0x10, 0x10}, + {HPRE_CORE10_BITMAP_CAP, "HPRE_CORE10_BITMAP_CAP ", 0x3170, 0x0, 0x10, 0x10}, }; static const struct hpre_hw_error hpre_hw_errors[] = { @@ -360,7 +370,7 @@ bool hpre_check_alg_support(struct hisi_qm *qm, u32 alg) { u32 cap_val; - cap_val = qm->cap_tables.dev_cap_table[HPRE_DRV_ALG_BITMAP_CAP_IDX].cap_val; + cap_val = qm->cap_tables.dev_cap_table[HPRE_DRV_ALG_BITMAP].cap_val; if (alg & cap_val) return true; @@ -415,7 +425,7 @@ static int pf_q_num_set(const char *val, const struct kernel_param *kp) { pf_q_num_flag = true; - return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_HPRE_PF); + return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_HPRE_PF); } static const struct kernel_param_ops hpre_pf_q_num_ops = { @@ -503,14 +513,17 @@ static int hpre_cfg_by_dsm(struct hisi_qm *qm) static int hpre_set_cluster(struct hisi_qm *qm) { struct device *dev = &qm->pdev->dev; - unsigned long offset; u32 cluster_core_mask; + unsigned long offset; + u32 hpre_core_info; u8 clusters_num; u32 val = 0; int ret, i; - cluster_core_mask = qm->cap_tables.dev_cap_table[HPRE_CORE_ENABLE_BITMAP_CAP_IDX].cap_val; - clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; + cluster_core_mask = qm->cap_tables.dev_cap_table[HPRE_CORE_EN].cap_val; + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; for (i = 0; i < clusters_num; i++) { offset = i * HPRE_CLSTR_ADDR_INTRVL; @@ -593,6 +606,9 @@ static void hpre_close_sva_prefetch(struct hisi_qm *qm) static void hpre_enable_clock_gate(struct hisi_qm *qm) { + unsigned long offset; + u8 clusters_num, i; + u32 hpre_core_info; u32 val; if (qm->ver < QM_HW_V3) @@ -606,17 +622,26 @@ static void hpre_enable_clock_gate(struct hisi_qm *qm) val |= HPRE_PEH_CFG_AUTO_GATE_EN; writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE); - val = readl(qm->io_base + HPRE_CLUSTER_DYN_CTL); - val |= HPRE_CLUSTER_DYN_CTL_EN; - writel(val, qm->io_base + HPRE_CLUSTER_DYN_CTL); - - val = readl_relaxed(qm->io_base + HPRE_CORE_SHB_CFG); - val |= HPRE_CORE_GATE_EN; - writel(val, qm->io_base + HPRE_CORE_SHB_CFG); + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL; + val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + val |= HPRE_CLUSTER_DYN_CTL_EN; + writel(val, qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + + val = readl(qm->io_base + offset + HPRE_CORE_SHB_CFG); + val |= HPRE_CORE_GATE_EN; + writel(val, qm->io_base + offset + HPRE_CORE_SHB_CFG); + } } static void hpre_disable_clock_gate(struct hisi_qm *qm) { + unsigned long offset; + u8 clusters_num, i; + u32 hpre_core_info; u32 val; if (qm->ver < QM_HW_V3) @@ -630,13 +655,19 @@ static void hpre_disable_clock_gate(struct hisi_qm *qm) val &= ~HPRE_PEH_CFG_AUTO_GATE_EN; writel(val, qm->io_base + HPRE_PEH_CFG_AUTO_GATE); - val = readl(qm->io_base + HPRE_CLUSTER_DYN_CTL); - val &= ~HPRE_CLUSTER_DYN_CTL_EN; - writel(val, qm->io_base + HPRE_CLUSTER_DYN_CTL); - - val = readl_relaxed(qm->io_base + HPRE_CORE_SHB_CFG); - val &= ~HPRE_CORE_GATE_EN; - writel(val, qm->io_base + HPRE_CORE_SHB_CFG); + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + for (i = 0; i < clusters_num; i++) { + offset = (unsigned long)i * HPRE_CLSTR_ADDR_INTRVL; + val = readl(qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + val &= ~HPRE_CLUSTER_DYN_CTL_EN; + writel(val, qm->io_base + offset + HPRE_CLUSTER_DYN_CTL); + + val = readl(qm->io_base + offset + HPRE_CORE_SHB_CFG); + val &= ~HPRE_CORE_GATE_EN; + writel(val, qm->io_base + offset + HPRE_CORE_SHB_CFG); + } } static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) @@ -699,11 +730,14 @@ static int hpre_set_user_domain_and_cache(struct hisi_qm *qm) static void hpre_cnt_regs_clear(struct hisi_qm *qm) { unsigned long offset; + u32 hpre_core_info; u8 clusters_num; int i; /* clear clusterX/cluster_ctrl */ - clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; for (i = 0; i < clusters_num; i++) { offset = HPRE_CLSTR_BASE + i * HPRE_CLSTR_ADDR_INTRVL; writel(0x0, qm->io_base + offset + HPRE_CLUSTER_INQURY); @@ -995,10 +1029,13 @@ static int hpre_cluster_debugfs_init(struct hisi_qm *qm) char buf[HPRE_DBGFS_VAL_MAX_LEN]; struct debugfs_regset32 *regset; struct dentry *tmp_d; + u32 hpre_core_info; u8 clusters_num; int i, ret; - clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; for (i = 0; i < clusters_num; i++) { ret = snprintf(buf, HPRE_DBGFS_VAL_MAX_LEN, "cluster%d", i); if (ret >= HPRE_DBGFS_VAL_MAX_LEN) @@ -1041,6 +1078,26 @@ static int hpre_ctrl_debug_init(struct hisi_qm *qm) return hpre_cluster_debugfs_init(qm); } +static int hpre_cap_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 i, size; + + size = qm->cap_tables.qm_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, + qm->cap_tables.qm_cap_table[i].cap_val); + + size = qm->cap_tables.dev_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, + qm->cap_tables.dev_cap_table[i].cap_val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(hpre_cap_regs); + static void hpre_dfx_debug_init(struct hisi_qm *qm) { struct dfx_diff_registers *hpre_regs = qm->debug.acc_diff_regs; @@ -1059,6 +1116,9 @@ static void hpre_dfx_debug_init(struct hisi_qm *qm) if (qm->fun_type == QM_HW_PF && hpre_regs) debugfs_create_file("diff_regs", 0444, parent, qm, &hpre_diff_regs_fops); + + debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, + qm->debug.debug_root, qm, &hpre_cap_regs_fops); } static int hpre_debugfs_init(struct hisi_qm *qm) @@ -1106,26 +1166,33 @@ static int hpre_pre_store_cap_reg(struct hisi_qm *qm) { struct hisi_qm_cap_record *hpre_cap; struct device *dev = &qm->pdev->dev; + u32 hpre_core_info; + u8 clusters_num; size_t i, size; - size = ARRAY_SIZE(hpre_pre_store_caps); + size = ARRAY_SIZE(hpre_cap_query_info); hpre_cap = devm_kzalloc(dev, sizeof(*hpre_cap) * size, GFP_KERNEL); if (!hpre_cap) return -ENOMEM; for (i = 0; i < size; i++) { - hpre_cap[i].type = hpre_pre_store_caps[i]; - hpre_cap[i].cap_val = hisi_qm_get_hw_info(qm, hpre_basic_info, - hpre_pre_store_caps[i], qm->cap_ver); + hpre_cap[i].type = hpre_cap_query_info[i].type; + hpre_cap[i].name = hpre_cap_query_info[i].name; + hpre_cap[i].cap_val = hisi_qm_get_cap_value(qm, hpre_cap_query_info, + i, qm->cap_ver); } - if (hpre_cap[HPRE_CLUSTER_NUM_CAP_IDX].cap_val > HPRE_CLUSTERS_NUM_MAX) { + hpre_core_info = hpre_cap[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; + if (clusters_num > HPRE_CLUSTERS_NUM_MAX) { dev_err(dev, "Device cluster num %u is out of range for driver supports %d!\n", - hpre_cap[HPRE_CLUSTER_NUM_CAP_IDX].cap_val, HPRE_CLUSTERS_NUM_MAX); + clusters_num, HPRE_CLUSTERS_NUM_MAX); return -EINVAL; } qm->cap_tables.dev_cap_table = hpre_cap; + qm->cap_tables.dev_cap_size = size; return 0; } @@ -1172,7 +1239,7 @@ static int hpre_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; } - alg_msk = qm->cap_tables.dev_cap_table[HPRE_DEV_ALG_BITMAP_CAP_IDX].cap_val; + alg_msk = qm->cap_tables.dev_cap_table[HPRE_ALG_BITMAP].cap_val; ret = hisi_qm_set_algs(qm, alg_msk, hpre_dev_algs, ARRAY_SIZE(hpre_dev_algs)); if (ret) { pci_err(pdev, "Failed to set hpre algs!\n"); @@ -1188,10 +1255,13 @@ static int hpre_show_last_regs_init(struct hisi_qm *qm) int com_dfx_regs_num = ARRAY_SIZE(hpre_com_dfx_regs); struct qm_debug *debug = &qm->debug; void __iomem *io_base; + u32 hpre_core_info; u8 clusters_num; int i, j, idx; - clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; debug->last_words = kcalloc(cluster_dfx_regs_num * clusters_num + com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); if (!debug->last_words) @@ -1231,6 +1301,7 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm) struct qm_debug *debug = &qm->debug; struct pci_dev *pdev = qm->pdev; void __iomem *io_base; + u32 hpre_core_info; u8 clusters_num; int i, j, idx; u32 val; @@ -1246,7 +1317,9 @@ static void hpre_show_last_dfx_regs(struct hisi_qm *qm) hpre_com_dfx_regs[i].name, debug->last_words[i], val); } - clusters_num = qm->cap_tables.dev_cap_table[HPRE_CLUSTER_NUM_CAP_IDX].cap_val; + hpre_core_info = qm->cap_tables.dev_cap_table[HPRE_CORE_INFO].cap_val; + clusters_num = (hpre_core_info >> hpre_basic_info[HPRE_CLUSTER_NUM_CAP].shift) & + hpre_basic_info[HPRE_CLUSTER_NUM_CAP].mask; for (i = 0; i < clusters_num; i++) { io_base = qm->io_base + hpre_cluster_offsets[i]; for (j = 0; j < cluster_dfx_regs_num; j++) { @@ -1280,11 +1353,15 @@ static u32 hpre_get_hw_err_status(struct hisi_qm *qm) static void hpre_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { - u32 nfe; - writel(err_sts, qm->io_base + HPRE_HAC_SOURCE_INT); - nfe = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); - writel(nfe, qm->io_base + HPRE_RAS_NFE_ENB); +} + +static void hpre_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + u32 nfe_mask; + + nfe_mask = hisi_qm_get_hw_info(qm, hpre_basic_info, HPRE_NFE_MASK_CAP, qm->cap_ver); + writel(nfe_mask & (~err_type), qm->io_base + HPRE_RAS_NFE_ENB); } static void hpre_open_axi_master_ooo(struct hisi_qm *qm) @@ -1298,6 +1375,27 @@ static void hpre_open_axi_master_ooo(struct hisi_qm *qm) qm->io_base + HPRE_AM_OOO_SHUTDOWN_ENB); } +static enum acc_err_result hpre_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hpre_get_hw_err_status(qm); + if (err_status) { + if (err_status & qm->err_info.ecc_2bits_mask) + qm->err_status.is_dev_ecc_mbit = true; + hpre_log_hw_error(qm, err_status); + + if (err_status & qm->err_info.dev_reset_mask) { + /* Disable the same error reporting until device is recovered. */ + hpre_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + hpre_clear_hw_err_status(qm, err_status); + } + + return ACC_ERR_RECOVERED; +} + static void hpre_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -1324,12 +1422,12 @@ static const struct hisi_qm_err_ini hpre_err_ini = { .hw_err_disable = hpre_hw_error_disable, .get_dev_hw_err_status = hpre_get_hw_err_status, .clear_dev_hw_err_status = hpre_clear_hw_err_status, - .log_dev_hw_err = hpre_log_hw_error, .open_axi_master_ooo = hpre_open_axi_master_ooo, .open_sva_prefetch = hpre_open_sva_prefetch, .close_sva_prefetch = hpre_close_sva_prefetch, .show_last_dfx_regs = hpre_show_last_dfx_regs, .err_info_init = hpre_err_info_init, + .get_err_result = hpre_get_err_result, }; static int hpre_pf_probe_init(struct hpre *hpre) diff --git a/drivers/crypto/hisilicon/qm.c b/drivers/crypto/hisilicon/qm.c index 07983af9e3e2..19c1b5d3c954 100644 --- a/drivers/crypto/hisilicon/qm.c +++ b/drivers/crypto/hisilicon/qm.c @@ -271,12 +271,6 @@ enum vft_type { SHAPER_VFT, }; -enum acc_err_result { - ACC_ERR_NONE, - ACC_ERR_NEED_RESET, - ACC_ERR_RECOVERED, -}; - enum qm_alg_type { ALG_TYPE_0, ALG_TYPE_1, @@ -307,11 +301,29 @@ enum qm_basic_type { QM_VF_IRQ_NUM_CAP, }; -enum qm_pre_store_cap_idx { - QM_EQ_IRQ_TYPE_CAP_IDX = 0x0, - QM_AEQ_IRQ_TYPE_CAP_IDX, - QM_ABN_IRQ_TYPE_CAP_IDX, - QM_PF2VF_IRQ_TYPE_CAP_IDX, +enum qm_cap_table_type { + QM_CAP_VF = 0x0, + QM_AEQE_NUM, + QM_SCQE_NUM, + QM_EQ_IRQ, + QM_AEQ_IRQ, + QM_ABNORMAL_IRQ, + QM_MB_IRQ, + MAX_IRQ_NUM, + EXT_BAR_INDEX, +}; + +static const struct hisi_qm_cap_query_info qm_cap_query_info[] = { + {QM_CAP_VF, "QM_CAP_VF ", 0x3100, 0x0, 0x0, 0x6F01}, + {QM_AEQE_NUM, "QM_AEQE_NUM ", 0x3104, 0x800, 0x4000800, 0x4000800}, + {QM_SCQE_NUM, "QM_SCQE_NUM ", + 0x3108, 0x4000400, 0x4000400, 0x4000400}, + {QM_EQ_IRQ, "QM_EQ_IRQ ", 0x310c, 0x10000, 0x10000, 0x10000}, + {QM_AEQ_IRQ, "QM_AEQ_IRQ ", 0x3110, 0x0, 0x10001, 0x10001}, + {QM_ABNORMAL_IRQ, "QM_ABNORMAL_IRQ ", 0x3114, 0x0, 0x10003, 0x10003}, + {QM_MB_IRQ, "QM_MB_IRQ ", 0x3118, 0x0, 0x0, 0x10002}, + {MAX_IRQ_NUM, "MAX_IRQ_NUM ", 0x311c, 0x10001, 0x40002, 0x40003}, + {EXT_BAR_INDEX, "EXT_BAR_INDEX ", 0x3120, 0x0, 0x0, 0x14}, }; static const struct hisi_qm_cap_info qm_cap_info_comm[] = { @@ -344,13 +356,6 @@ static const struct hisi_qm_cap_info qm_basic_info[] = { {QM_VF_IRQ_NUM_CAP, 0x311c, 0, GENMASK(15, 0), 0x1, 0x2, 0x3}, }; -static const u32 qm_pre_store_caps[] = { - QM_EQ_IRQ_TYPE_CAP, - QM_AEQ_IRQ_TYPE_CAP, - QM_ABN_IRQ_TYPE_CAP, - QM_PF2VF_IRQ_TYPE_CAP, -}; - struct qm_mailbox { __le16 w0; __le16 queue_num; @@ -451,6 +456,37 @@ static struct qm_typical_qos_table shaper_cbs_s[] = { static void qm_irqs_unregister(struct hisi_qm *qm); static int qm_reset_device(struct hisi_qm *qm); +int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, + unsigned int device) +{ + struct pci_dev *pdev; + u32 n, q_num; + int ret; + + if (!val) + return -EINVAL; + + pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL); + if (!pdev) { + q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); + pr_info("No device found currently, suppose queue number is %u\n", + q_num); + } else { + if (pdev->revision == QM_HW_V1) + q_num = QM_QNUM_V1; + else + q_num = QM_QNUM_V2; + + pci_dev_put(pdev); + } + + ret = kstrtou32(val, 10, &n); + if (ret || n < QM_MIN_QNUM || n > q_num) + return -EINVAL; + + return param_set_int(val, kp); +} +EXPORT_SYMBOL_GPL(hisi_qm_q_num_set); static u32 qm_get_hw_error_status(struct hisi_qm *qm) { @@ -763,6 +799,27 @@ u32 hisi_qm_get_hw_info(struct hisi_qm *qm, } EXPORT_SYMBOL_GPL(hisi_qm_get_hw_info); +u32 hisi_qm_get_cap_value(struct hisi_qm *qm, + const struct hisi_qm_cap_query_info *info_table, + u32 index, bool is_read) +{ + u32 val; + + switch (qm->ver) { + case QM_HW_V1: + return info_table[index].v1_val; + case QM_HW_V2: + return info_table[index].v2_val; + default: + if (!is_read) + return info_table[index].v3_val; + + val = readl(qm->io_base + info_table[index].offset); + return val; + } +} +EXPORT_SYMBOL_GPL(hisi_qm_get_cap_value); + static void qm_get_xqc_depth(struct hisi_qm *qm, u16 *low_bits, u16 *high_bits, enum qm_basic_type type) { @@ -1425,22 +1482,25 @@ static void qm_log_hw_error(struct hisi_qm *qm, u32 error_status) static enum acc_err_result qm_hw_error_handle_v2(struct hisi_qm *qm) { - u32 error_status, tmp; - - /* read err sts */ - tmp = readl(qm->io_base + QM_ABNORMAL_INT_STATUS); - error_status = qm->error_mask & tmp; + u32 error_status; - if (error_status) { + error_status = qm_get_hw_error_status(qm); + if (error_status & qm->error_mask) { if (error_status & QM_ECC_MBIT) qm->err_status.is_qm_ecc_mbit = true; qm_log_hw_error(qm, error_status); - if (error_status & qm->err_info.qm_reset_mask) + if (error_status & qm->err_info.qm_reset_mask) { + /* Disable the same error reporting until device is recovered. */ + writel(qm->err_info.nfe & (~error_status), + qm->io_base + QM_RAS_NFE_ENABLE); return ACC_ERR_NEED_RESET; + } + /* Clear error source if not need reset. */ writel(error_status, qm->io_base + QM_ABNORMAL_INT_SOURCE); writel(qm->err_info.nfe, qm->io_base + QM_RAS_NFE_ENABLE); + writel(qm->err_info.ce, qm->io_base + QM_RAS_CE_ENABLE); } return ACC_ERR_RECOVERED; @@ -3861,30 +3921,12 @@ EXPORT_SYMBOL_GPL(hisi_qm_sriov_configure); static enum acc_err_result qm_dev_err_handle(struct hisi_qm *qm) { - u32 err_sts; - - if (!qm->err_ini->get_dev_hw_err_status) { - dev_err(&qm->pdev->dev, "Device doesn't support get hw error status!\n"); + if (!qm->err_ini->get_err_result) { + dev_err(&qm->pdev->dev, "Device doesn't support reset!\n"); return ACC_ERR_NONE; } - /* get device hardware error status */ - err_sts = qm->err_ini->get_dev_hw_err_status(qm); - if (err_sts) { - if (err_sts & qm->err_info.ecc_2bits_mask) - qm->err_status.is_dev_ecc_mbit = true; - - if (qm->err_ini->log_dev_hw_err) - qm->err_ini->log_dev_hw_err(qm, err_sts); - - if (err_sts & qm->err_info.dev_reset_mask) - return ACC_ERR_NEED_RESET; - - if (qm->err_ini->clear_dev_hw_err_status) - qm->err_ini->clear_dev_hw_err_status(qm, err_sts); - } - - return ACC_ERR_RECOVERED; + return qm->err_ini->get_err_result(qm); } static enum acc_err_result qm_process_dev_error(struct hisi_qm *qm) @@ -4866,7 +4908,7 @@ static void qm_unregister_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return; - val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return; @@ -4883,7 +4925,7 @@ static int qm_register_abnormal_irq(struct hisi_qm *qm) if (qm->fun_type == QM_HW_VF) return 0; - val = qm->cap_tables.qm_cap_table[QM_ABN_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_ABNORMAL_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_ABN_IRQ_TYPE_MASK)) return 0; @@ -4900,7 +4942,7 @@ static void qm_unregister_mb_cmd_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; @@ -4914,7 +4956,7 @@ static int qm_register_mb_cmd_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - val = qm->cap_tables.qm_cap_table[QM_PF2VF_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_MB_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return 0; @@ -4931,7 +4973,7 @@ static void qm_unregister_aeq_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; @@ -4945,7 +4987,7 @@ static int qm_register_aeq_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_AEQ_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return 0; @@ -4963,7 +5005,7 @@ static void qm_unregister_eq_irq(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; u32 irq_vector, val; - val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return; @@ -4977,7 +5019,7 @@ static int qm_register_eq_irq(struct hisi_qm *qm) u32 irq_vector, val; int ret; - val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ_TYPE_CAP_IDX].cap_val; + val = qm->cap_tables.qm_cap_table[QM_EQ_IRQ].cap_val; if (!((val >> QM_IRQ_TYPE_SHIFT) & QM_IRQ_TYPE_MASK)) return 0; @@ -5065,24 +5107,26 @@ static int qm_get_qp_num(struct hisi_qm *qm) return 0; } -static int qm_pre_store_irq_type_caps(struct hisi_qm *qm) +static int qm_pre_store_caps(struct hisi_qm *qm) { struct hisi_qm_cap_record *qm_cap; struct pci_dev *pdev = qm->pdev; size_t i, size; - size = ARRAY_SIZE(qm_pre_store_caps); + size = ARRAY_SIZE(qm_cap_query_info); qm_cap = devm_kzalloc(&pdev->dev, sizeof(*qm_cap) * size, GFP_KERNEL); if (!qm_cap) return -ENOMEM; for (i = 0; i < size; i++) { - qm_cap[i].type = qm_pre_store_caps[i]; - qm_cap[i].cap_val = hisi_qm_get_hw_info(qm, qm_basic_info, - qm_pre_store_caps[i], qm->cap_ver); + qm_cap[i].type = qm_cap_query_info[i].type; + qm_cap[i].name = qm_cap_query_info[i].name; + qm_cap[i].cap_val = hisi_qm_get_cap_value(qm, qm_cap_query_info, + i, qm->cap_ver); } qm->cap_tables.qm_cap_table = qm_cap; + qm->cap_tables.qm_cap_size = size; return 0; } @@ -5119,8 +5163,8 @@ static int qm_get_hw_caps(struct hisi_qm *qm) set_bit(cap_info[i].type, &qm->caps); } - /* Fetch and save the value of irq type related capability registers */ - return qm_pre_store_irq_type_caps(qm); + /* Fetch and save the value of qm capability registers */ + return qm_pre_store_caps(qm); } static int qm_get_pci_res(struct hisi_qm *qm) diff --git a/drivers/crypto/hisilicon/sec/sec_drv.c b/drivers/crypto/hisilicon/sec/sec_drv.c index 9bafcc5aa404..ef0cb733c92c 100644 --- a/drivers/crypto/hisilicon/sec/sec_drv.c +++ b/drivers/crypto/hisilicon/sec/sec_drv.c @@ -1304,7 +1304,7 @@ MODULE_DEVICE_TABLE(acpi, sec_acpi_match); static struct platform_driver sec_driver = { .probe = sec_probe, - .remove_new = sec_remove, + .remove = sec_remove, .driver = { .name = "hisi_sec_platform_driver", .of_match_table = sec_match, diff --git a/drivers/crypto/hisilicon/sec2/sec.h b/drivers/crypto/hisilicon/sec2/sec.h index 410c83712e28..356188bee6fb 100644 --- a/drivers/crypto/hisilicon/sec2/sec.h +++ b/drivers/crypto/hisilicon/sec2/sec.h @@ -220,11 +220,27 @@ enum sec_cap_type { SEC_CORE4_ALG_BITMAP_HIGH, }; -enum sec_cap_reg_record_idx { - SEC_DRV_ALG_BITMAP_LOW_IDX = 0x0, - SEC_DRV_ALG_BITMAP_HIGH_IDX, - SEC_DEV_ALG_BITMAP_LOW_IDX, - SEC_DEV_ALG_BITMAP_HIGH_IDX, +enum sec_cap_table_type { + QM_RAS_NFE_TYPE = 0x0, + QM_RAS_NFE_RESET, + QM_RAS_CE_TYPE, + SEC_RAS_NFE_TYPE, + SEC_RAS_NFE_RESET, + SEC_RAS_CE_TYPE, + SEC_CORE_INFO, + SEC_CORE_EN, + SEC_DRV_ALG_BITMAP_LOW_TB, + SEC_DRV_ALG_BITMAP_HIGH_TB, + SEC_ALG_BITMAP_LOW, + SEC_ALG_BITMAP_HIGH, + SEC_CORE1_BITMAP_LOW, + SEC_CORE1_BITMAP_HIGH, + SEC_CORE2_BITMAP_LOW, + SEC_CORE2_BITMAP_HIGH, + SEC_CORE3_BITMAP_LOW, + SEC_CORE3_BITMAP_HIGH, + SEC_CORE4_BITMAP_LOW, + SEC_CORE4_BITMAP_HIGH, }; void sec_destroy_qps(struct hisi_qp **qps, int qp_num); diff --git a/drivers/crypto/hisilicon/sec2/sec_crypto.c b/drivers/crypto/hisilicon/sec2/sec_crypto.c index 0558f98e221f..ae9ebbb4103d 100644 --- a/drivers/crypto/hisilicon/sec2/sec_crypto.c +++ b/drivers/crypto/hisilicon/sec2/sec_crypto.c @@ -2520,8 +2520,8 @@ int sec_register_to_crypto(struct hisi_qm *qm) u64 alg_mask; int ret = 0; - alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, - SEC_DRV_ALG_BITMAP_LOW_IDX); + alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, + SEC_DRV_ALG_BITMAP_LOW_TB); mutex_lock(&sec_algs_lock); if (sec_available_devs) { @@ -2553,8 +2553,8 @@ void sec_unregister_from_crypto(struct hisi_qm *qm) { u64 alg_mask; - alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_IDX, - SEC_DRV_ALG_BITMAP_LOW_IDX); + alg_mask = sec_get_alg_bitmap(qm, SEC_DRV_ALG_BITMAP_HIGH_TB, + SEC_DRV_ALG_BITMAP_LOW_TB); mutex_lock(&sec_algs_lock); if (--sec_available_devs) diff --git a/drivers/crypto/hisilicon/sec2/sec_main.c b/drivers/crypto/hisilicon/sec2/sec_main.c index c35533d8930b..8ec5333bb5aa 100644 --- a/drivers/crypto/hisilicon/sec2/sec_main.c +++ b/drivers/crypto/hisilicon/sec2/sec_main.c @@ -14,9 +14,9 @@ #include <linux/seq_file.h> #include <linux/topology.h> #include <linux/uacce.h> - #include "sec.h" +#define CAP_FILE_PERMISSION 0444 #define SEC_VF_NUM 63 #define SEC_QUEUE_NUM_V1 4096 #define PCI_DEVICE_ID_HUAWEI_SEC_PF 0xa255 @@ -167,11 +167,34 @@ static const struct hisi_qm_cap_info sec_basic_info[] = { {SEC_CORE4_ALG_BITMAP_HIGH, 0x3170, 0, GENMASK(31, 0), 0x3FFF, 0x3FFF, 0x3FFF}, }; -static const u32 sec_pre_store_caps[] = { - SEC_DRV_ALG_BITMAP_LOW, - SEC_DRV_ALG_BITMAP_HIGH, - SEC_DEV_ALG_BITMAP_LOW, - SEC_DEV_ALG_BITMAP_HIGH, +static const struct hisi_qm_cap_query_info sec_cap_query_info[] = { + {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C77, 0x7C77}, + {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC77, 0x6C77}, + {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, + {SEC_RAS_NFE_TYPE, "SEC_RAS_NFE_TYPE ", 0x3130, 0x0, 0x177, 0x60177}, + {SEC_RAS_NFE_RESET, "SEC_RAS_NFE_RESET ", 0x3134, 0x0, 0x177, 0x177}, + {SEC_RAS_CE_TYPE, "SEC_RAS_CE_TYPE ", 0x3138, 0x0, 0x88, 0xC088}, + {SEC_CORE_INFO, "SEC_CORE_INFO ", 0x313c, 0x110404, 0x110404, 0x110404}, + {SEC_CORE_EN, "SEC_CORE_EN ", 0x3140, 0x17F, 0x17F, 0xF}, + {SEC_DRV_ALG_BITMAP_LOW_TB, "SEC_DRV_ALG_BITMAP_LOW ", + 0x3144, 0x18050CB, 0x18050CB, 0x18670CF}, + {SEC_DRV_ALG_BITMAP_HIGH_TB, "SEC_DRV_ALG_BITMAP_HIGH ", + 0x3148, 0x395C, 0x395C, 0x395C}, + {SEC_ALG_BITMAP_LOW, "SEC_ALG_BITMAP_LOW ", + 0x314c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_ALG_BITMAP_HIGH, "SEC_ALG_BITMAP_HIGH ", 0x3150, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE1_BITMAP_LOW, "SEC_CORE1_BITMAP_LOW ", + 0x3154, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE1_BITMAP_HIGH, "SEC_CORE1_BITMAP_HIGH ", 0x3158, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE2_BITMAP_LOW, "SEC_CORE2_BITMAP_LOW ", + 0x315c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE2_BITMAP_HIGH, "SEC_CORE2_BITMAP_HIGH ", 0x3160, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE3_BITMAP_LOW, "SEC_CORE3_BITMAP_LOW ", + 0x3164, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE3_BITMAP_HIGH, "SEC_CORE3_BITMAP_HIGH ", 0x3168, 0x3FFF, 0x3FFF, 0x3FFF}, + {SEC_CORE4_BITMAP_LOW, "SEC_CORE4_BITMAP_LOW ", + 0x316c, 0xFFFFFFFF, 0xFFFFFFFF, 0xFFFFFFFF}, + {SEC_CORE4_BITMAP_HIGH, "SEC_CORE4_BITMAP_HIGH ", 0x3170, 0x3FFF, 0x3FFF, 0x3FFF}, }; static const struct qm_dev_alg sec_dev_algs[] = { { @@ -322,7 +345,7 @@ static int sec_pf_q_num_set(const char *val, const struct kernel_param *kp) { pf_q_num_flag = true; - return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_SEC_PF); + return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_SEC_PF); } static const struct kernel_param_ops sec_pf_q_num_ops = { @@ -838,6 +861,26 @@ static int sec_regs_show(struct seq_file *s, void *unused) DEFINE_SHOW_ATTRIBUTE(sec_regs); +static int sec_cap_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 i, size; + + size = qm->cap_tables.qm_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, + qm->cap_tables.qm_cap_table[i].cap_val); + + size = qm->cap_tables.dev_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, + qm->cap_tables.dev_cap_table[i].cap_val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(sec_cap_regs); + static int sec_core_debug_init(struct hisi_qm *qm) { struct dfx_diff_registers *sec_regs = qm->debug.acc_diff_regs; @@ -872,6 +915,9 @@ static int sec_core_debug_init(struct hisi_qm *qm) tmp_d, data, &sec_atomic64_ops); } + debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, + qm->debug.debug_root, qm, &sec_cap_regs_fops); + return 0; } @@ -1010,11 +1056,15 @@ static u32 sec_get_hw_err_status(struct hisi_qm *qm) static void sec_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { - u32 nfe; - writel(err_sts, qm->io_base + SEC_CORE_INT_SOURCE); - nfe = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); - writel(nfe, qm->io_base + SEC_RAS_NFE_REG); +} + +static void sec_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + u32 nfe_mask; + + nfe_mask = hisi_qm_get_hw_info(qm, sec_basic_info, SEC_NFE_MASK_CAP, qm->cap_ver); + writel(nfe_mask & (~err_type), qm->io_base + SEC_RAS_NFE_REG); } static void sec_open_axi_master_ooo(struct hisi_qm *qm) @@ -1026,6 +1076,27 @@ static void sec_open_axi_master_ooo(struct hisi_qm *qm) writel(val | SEC_AXI_SHUTDOWN_ENABLE, qm->io_base + SEC_CONTROL_REG); } +static enum acc_err_result sec_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = sec_get_hw_err_status(qm); + if (err_status) { + if (err_status & qm->err_info.ecc_2bits_mask) + qm->err_status.is_dev_ecc_mbit = true; + sec_log_hw_error(qm, err_status); + + if (err_status & qm->err_info.dev_reset_mask) { + /* Disable the same error reporting until device is recovered. */ + sec_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + sec_clear_hw_err_status(qm, err_status); + } + + return ACC_ERR_RECOVERED; +} + static void sec_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -1052,12 +1123,12 @@ static const struct hisi_qm_err_ini sec_err_ini = { .hw_err_disable = sec_hw_error_disable, .get_dev_hw_err_status = sec_get_hw_err_status, .clear_dev_hw_err_status = sec_clear_hw_err_status, - .log_dev_hw_err = sec_log_hw_error, .open_axi_master_ooo = sec_open_axi_master_ooo, .open_sva_prefetch = sec_open_sva_prefetch, .close_sva_prefetch = sec_close_sva_prefetch, .show_last_dfx_regs = sec_show_last_dfx_regs, .err_info_init = sec_err_info_init, + .get_err_result = sec_get_err_result, }; static int sec_pf_probe_init(struct sec_dev *sec) @@ -1085,18 +1156,20 @@ static int sec_pre_store_cap_reg(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; size_t i, size; - size = ARRAY_SIZE(sec_pre_store_caps); + size = ARRAY_SIZE(sec_cap_query_info); sec_cap = devm_kzalloc(&pdev->dev, sizeof(*sec_cap) * size, GFP_KERNEL); if (!sec_cap) return -ENOMEM; for (i = 0; i < size; i++) { - sec_cap[i].type = sec_pre_store_caps[i]; - sec_cap[i].cap_val = hisi_qm_get_hw_info(qm, sec_basic_info, - sec_pre_store_caps[i], qm->cap_ver); + sec_cap[i].type = sec_cap_query_info[i].type; + sec_cap[i].name = sec_cap_query_info[i].name; + sec_cap[i].cap_val = hisi_qm_get_cap_value(qm, sec_cap_query_info, + i, qm->cap_ver); } qm->cap_tables.dev_cap_table = sec_cap; + qm->cap_tables.dev_cap_size = size; return 0; } @@ -1146,8 +1219,7 @@ static int sec_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) hisi_qm_uninit(qm); return ret; } - - alg_msk = sec_get_alg_bitmap(qm, SEC_DEV_ALG_BITMAP_HIGH_IDX, SEC_DEV_ALG_BITMAP_LOW_IDX); + alg_msk = sec_get_alg_bitmap(qm, SEC_ALG_BITMAP_HIGH, SEC_ALG_BITMAP_LOW); ret = hisi_qm_set_algs(qm, alg_msk, sec_dev_algs, ARRAY_SIZE(sec_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set sec algs!\n"); diff --git a/drivers/crypto/hisilicon/trng/trng.c b/drivers/crypto/hisilicon/trng/trng.c index 66c551ecdee8..ac74df4a9471 100644 --- a/drivers/crypto/hisilicon/trng/trng.c +++ b/drivers/crypto/hisilicon/trng/trng.c @@ -324,7 +324,7 @@ MODULE_DEVICE_TABLE(acpi, hisi_trng_acpi_match); static struct platform_driver hisi_trng_driver = { .probe = hisi_trng_probe, - .remove_new = hisi_trng_remove, + .remove = hisi_trng_remove, .driver = { .name = "hisi-trng-v2", .acpi_match_table = ACPI_PTR(hisi_trng_acpi_match), diff --git a/drivers/crypto/hisilicon/zip/zip.h b/drivers/crypto/hisilicon/zip/zip.h index f2e6da3240ae..2fecf346c3c9 100644 --- a/drivers/crypto/hisilicon/zip/zip.h +++ b/drivers/crypto/hisilicon/zip/zip.h @@ -81,6 +81,24 @@ struct hisi_zip_sqe { u32 rsvd1[4]; }; +enum zip_cap_table_type { + QM_RAS_NFE_TYPE, + QM_RAS_NFE_RESET, + QM_RAS_CE_TYPE, + ZIP_RAS_NFE_TYPE, + ZIP_RAS_NFE_RESET, + ZIP_RAS_CE_TYPE, + ZIP_CORE_INFO, + ZIP_CORE_EN, + ZIP_DRV_ALG_BITMAP_TB, + ZIP_ALG_BITMAP, + ZIP_CORE1_BITMAP, + ZIP_CORE2_BITMAP, + ZIP_CORE3_BITMAP, + ZIP_CORE4_BITMAP, + ZIP_CORE5_BITMAP, +}; + int zip_create_qps(struct hisi_qp **qps, int qp_num, int node); int hisi_zip_register_to_crypto(struct hisi_qm *qm); void hisi_zip_unregister_from_crypto(struct hisi_qm *qm); diff --git a/drivers/crypto/hisilicon/zip/zip_main.c b/drivers/crypto/hisilicon/zip/zip_main.c index d07e47b48be0..9239b251c2d7 100644 --- a/drivers/crypto/hisilicon/zip/zip_main.c +++ b/drivers/crypto/hisilicon/zip/zip_main.c @@ -14,6 +14,7 @@ #include <linux/uacce.h> #include "zip.h" +#define CAP_FILE_PERMISSION 0444 #define PCI_DEVICE_ID_HUAWEI_ZIP_PF 0xa250 #define HZIP_QUEUE_NUM_V1 4096 @@ -250,24 +251,22 @@ static struct hisi_qm_cap_info zip_basic_cap_info[] = { {ZIP_CAP_MAX, 0x317c, 0, GENMASK(0, 0), 0x0, 0x0, 0x0} }; -enum zip_pre_store_cap_idx { - ZIP_CORE_NUM_CAP_IDX = 0x0, - ZIP_CLUSTER_COMP_NUM_CAP_IDX, - ZIP_CLUSTER_DECOMP_NUM_CAP_IDX, - ZIP_DECOMP_ENABLE_BITMAP_IDX, - ZIP_COMP_ENABLE_BITMAP_IDX, - ZIP_DRV_ALG_BITMAP_IDX, - ZIP_DEV_ALG_BITMAP_IDX, -}; - -static const u32 zip_pre_store_caps[] = { - ZIP_CORE_NUM_CAP, - ZIP_CLUSTER_COMP_NUM_CAP, - ZIP_CLUSTER_DECOMP_NUM_CAP, - ZIP_DECOMP_ENABLE_BITMAP, - ZIP_COMP_ENABLE_BITMAP, - ZIP_DRV_ALG_BITMAP, - ZIP_DEV_ALG_BITMAP, +static const struct hisi_qm_cap_query_info zip_cap_query_info[] = { + {QM_RAS_NFE_TYPE, "QM_RAS_NFE_TYPE ", 0x3124, 0x0, 0x1C57, 0x7C77}, + {QM_RAS_NFE_RESET, "QM_RAS_NFE_RESET ", 0x3128, 0x0, 0xC57, 0x6C77}, + {QM_RAS_CE_TYPE, "QM_RAS_CE_TYPE ", 0x312C, 0x0, 0x8, 0x8}, + {ZIP_RAS_NFE_TYPE, "ZIP_RAS_NFE_TYPE ", 0x3130, 0x0, 0x7FE, 0x1FFE}, + {ZIP_RAS_NFE_RESET, "ZIP_RAS_NFE_RESET ", 0x3134, 0x0, 0x7FE, 0x7FE}, + {ZIP_RAS_CE_TYPE, "ZIP_RAS_CE_TYPE ", 0x3138, 0x0, 0x1, 0x1}, + {ZIP_CORE_INFO, "ZIP_CORE_INFO ", 0x313C, 0x12080206, 0x12080206, 0x12050203}, + {ZIP_CORE_EN, "ZIP_CORE_EN ", 0x3140, 0xFC0003, 0xFC0003, 0x1C0003}, + {ZIP_DRV_ALG_BITMAP_TB, "ZIP_DRV_ALG_BITMAP ", 0x3144, 0x0, 0x0, 0x30}, + {ZIP_ALG_BITMAP, "ZIP_ALG_BITMAP ", 0x3148, 0xF, 0xF, 0x3F}, + {ZIP_CORE1_BITMAP, "ZIP_CORE1_BITMAP ", 0x314C, 0x5, 0x5, 0xD5}, + {ZIP_CORE2_BITMAP, "ZIP_CORE2_BITMAP ", 0x3150, 0x5, 0x5, 0xD5}, + {ZIP_CORE3_BITMAP, "ZIP_CORE3_BITMAP ", 0x3154, 0xA, 0xA, 0x2A}, + {ZIP_CORE4_BITMAP, "ZIP_CORE4_BITMAP ", 0x3158, 0xA, 0xA, 0x2A}, + {ZIP_CORE5_BITMAP, "ZIP_CORE5_BITMAP ", 0x315C, 0xA, 0xA, 0x2A}, }; static const struct debugfs_reg32 hzip_dfx_regs[] = { @@ -402,7 +401,7 @@ static int pf_q_num_set(const char *val, const struct kernel_param *kp) { pf_q_num_flag = true; - return q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_ZIP_PF); + return hisi_qm_q_num_set(val, kp, PCI_DEVICE_ID_HUAWEI_ZIP_PF); } static const struct kernel_param_ops pf_q_num_ops = { @@ -442,7 +441,7 @@ bool hisi_zip_alg_support(struct hisi_qm *qm, u32 alg) { u32 cap_val; - cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_IDX].cap_val; + cap_val = qm->cap_tables.dev_cap_table[ZIP_DRV_ALG_BITMAP_TB].cap_val; if ((alg & cap_val) == alg) return true; @@ -530,6 +529,7 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) { void __iomem *base = qm->io_base; u32 dcomp_bm, comp_bm; + u32 zip_core_en; /* qm user domain */ writel(AXUSER_BASE, base + QM_ARUSER_M_CFG_1); @@ -567,8 +567,12 @@ static int hisi_zip_set_user_domain_and_cache(struct hisi_qm *qm) } /* let's open all compression/decompression cores */ - dcomp_bm = qm->cap_tables.dev_cap_table[ZIP_DECOMP_ENABLE_BITMAP_IDX].cap_val; - comp_bm = qm->cap_tables.dev_cap_table[ZIP_COMP_ENABLE_BITMAP_IDX].cap_val; + + zip_core_en = qm->cap_tables.dev_cap_table[ZIP_CORE_EN].cap_val; + dcomp_bm = (zip_core_en >> zip_basic_cap_info[ZIP_DECOMP_ENABLE_BITMAP].shift) & + zip_basic_cap_info[ZIP_DECOMP_ENABLE_BITMAP].mask; + comp_bm = (zip_core_en >> zip_basic_cap_info[ZIP_COMP_ENABLE_BITMAP].shift) & + zip_basic_cap_info[ZIP_COMP_ENABLE_BITMAP].mask; writel(HZIP_DECOMP_CHECK_ENABLE | dcomp_bm | comp_bm, base + HZIP_CLOCK_GATE_CTRL); /* enable sqc,cqc writeback */ @@ -788,7 +792,12 @@ DEFINE_SHOW_ATTRIBUTE(hisi_zip_regs); static void __iomem *get_zip_core_addr(struct hisi_qm *qm, int core_num) { - u32 zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val; + u8 zip_comp_core_num; + u32 zip_core_info; + + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask; if (core_num < zip_comp_core_num) return qm->io_base + HZIP_CORE_DFX_BASE + @@ -803,12 +812,16 @@ static int hisi_zip_core_debug_init(struct hisi_qm *qm) u32 zip_core_num, zip_comp_core_num; struct device *dev = &qm->pdev->dev; struct debugfs_regset32 *regset; + u32 zip_core_info; struct dentry *tmp_d; char buf[HZIP_BUF_SIZE]; int i; - zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; - zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val; + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; + zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask; for (i = 0; i < zip_core_num; i++) { if (i < zip_comp_core_num) @@ -834,6 +847,26 @@ static int hisi_zip_core_debug_init(struct hisi_qm *qm) return 0; } +static int zip_cap_regs_show(struct seq_file *s, void *unused) +{ + struct hisi_qm *qm = s->private; + u32 i, size; + + size = qm->cap_tables.qm_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.qm_cap_table[i].name, + qm->cap_tables.qm_cap_table[i].cap_val); + + size = qm->cap_tables.dev_cap_size; + for (i = 0; i < size; i++) + seq_printf(s, "%s= 0x%08x\n", qm->cap_tables.dev_cap_table[i].name, + qm->cap_tables.dev_cap_table[i].cap_val); + + return 0; +} + +DEFINE_SHOW_ATTRIBUTE(zip_cap_regs); + static void hisi_zip_dfx_debug_init(struct hisi_qm *qm) { struct dfx_diff_registers *hzip_regs = qm->debug.acc_diff_regs; @@ -854,6 +887,9 @@ static void hisi_zip_dfx_debug_init(struct hisi_qm *qm) if (qm->fun_type == QM_HW_PF && hzip_regs) debugfs_create_file("diff_regs", 0444, tmp_dir, qm, &hzip_diff_regs_fops); + + debugfs_create_file("cap_regs", CAP_FILE_PERMISSION, + qm->debug.debug_root, qm, &zip_cap_regs_fops); } static int hisi_zip_ctrl_debug_init(struct hisi_qm *qm) @@ -912,9 +948,14 @@ debugfs_remove: /* hisi_zip_debug_regs_clear() - clear the zip debug regs */ static void hisi_zip_debug_regs_clear(struct hisi_qm *qm) { - u32 zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; + u32 zip_core_info; + u8 zip_core_num; int i, j; + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; + /* enable register read_clear bit */ writel(HZIP_RD_CNT_CLR_CE_EN, qm->io_base + HZIP_SOFT_CTRL_CNT_CLR_CE); for (i = 0; i < zip_core_num; i++) @@ -946,10 +987,13 @@ static int hisi_zip_show_last_regs_init(struct hisi_qm *qm) int com_dfx_regs_num = ARRAY_SIZE(hzip_com_dfx_regs); struct qm_debug *debug = &qm->debug; void __iomem *io_base; + u32 zip_core_info; u32 zip_core_num; int i, j, idx; - zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; debug->last_words = kcalloc(core_dfx_regs_num * zip_core_num + com_dfx_regs_num, sizeof(unsigned int), GFP_KERNEL); @@ -991,6 +1035,7 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) u32 zip_core_num, zip_comp_core_num; struct qm_debug *debug = &qm->debug; char buf[HZIP_BUF_SIZE]; + u32 zip_core_info; void __iomem *base; int i, j, idx; u32 val; @@ -1005,8 +1050,11 @@ static void hisi_zip_show_last_dfx_regs(struct hisi_qm *qm) hzip_com_dfx_regs[i].name, debug->last_words[i], val); } - zip_core_num = qm->cap_tables.dev_cap_table[ZIP_CORE_NUM_CAP_IDX].cap_val; - zip_comp_core_num = qm->cap_tables.dev_cap_table[ZIP_CLUSTER_COMP_NUM_CAP_IDX].cap_val; + zip_core_info = qm->cap_tables.dev_cap_table[ZIP_CORE_INFO].cap_val; + zip_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CORE_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CORE_NUM_CAP].mask; + zip_comp_core_num = (zip_core_info >> zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].shift) & + zip_basic_cap_info[ZIP_CLUSTER_COMP_NUM_CAP].mask; for (i = 0; i < zip_core_num; i++) { if (i < zip_comp_core_num) @@ -1059,11 +1107,15 @@ static u32 hisi_zip_get_hw_err_status(struct hisi_qm *qm) static void hisi_zip_clear_hw_err_status(struct hisi_qm *qm, u32 err_sts) { - u32 nfe; - writel(err_sts, qm->io_base + HZIP_CORE_INT_SOURCE); - nfe = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); - writel(nfe, qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); +} + +static void hisi_zip_disable_error_report(struct hisi_qm *qm, u32 err_type) +{ + u32 nfe_mask; + + nfe_mask = hisi_qm_get_hw_info(qm, zip_basic_cap_info, ZIP_NFE_MASK_CAP, qm->cap_ver); + writel(nfe_mask & (~err_type), qm->io_base + HZIP_CORE_INT_RAS_NFE_ENB); } static void hisi_zip_open_axi_master_ooo(struct hisi_qm *qm) @@ -1093,6 +1145,27 @@ static void hisi_zip_close_axi_master_ooo(struct hisi_qm *qm) qm->io_base + HZIP_CORE_INT_SET); } +static enum acc_err_result hisi_zip_get_err_result(struct hisi_qm *qm) +{ + u32 err_status; + + err_status = hisi_zip_get_hw_err_status(qm); + if (err_status) { + if (err_status & qm->err_info.ecc_2bits_mask) + qm->err_status.is_dev_ecc_mbit = true; + hisi_zip_log_hw_error(qm, err_status); + + if (err_status & qm->err_info.dev_reset_mask) { + /* Disable the same error reporting until device is recovered. */ + hisi_zip_disable_error_report(qm, err_status); + return ACC_ERR_NEED_RESET; + } + hisi_zip_clear_hw_err_status(qm, err_status); + } + + return ACC_ERR_RECOVERED; +} + static void hisi_zip_err_info_init(struct hisi_qm *qm) { struct hisi_qm_err_info *err_info = &qm->err_info; @@ -1120,13 +1193,13 @@ static const struct hisi_qm_err_ini hisi_zip_err_ini = { .hw_err_disable = hisi_zip_hw_error_disable, .get_dev_hw_err_status = hisi_zip_get_hw_err_status, .clear_dev_hw_err_status = hisi_zip_clear_hw_err_status, - .log_dev_hw_err = hisi_zip_log_hw_error, .open_axi_master_ooo = hisi_zip_open_axi_master_ooo, .close_axi_master_ooo = hisi_zip_close_axi_master_ooo, .open_sva_prefetch = hisi_zip_open_sva_prefetch, .close_sva_prefetch = hisi_zip_close_sva_prefetch, .show_last_dfx_regs = hisi_zip_show_last_dfx_regs, .err_info_init = hisi_zip_err_info_init, + .get_err_result = hisi_zip_get_err_result, }; static int hisi_zip_pf_probe_init(struct hisi_zip *hisi_zip) @@ -1167,18 +1240,20 @@ static int zip_pre_store_cap_reg(struct hisi_qm *qm) struct pci_dev *pdev = qm->pdev; size_t i, size; - size = ARRAY_SIZE(zip_pre_store_caps); + size = ARRAY_SIZE(zip_cap_query_info); zip_cap = devm_kzalloc(&pdev->dev, sizeof(*zip_cap) * size, GFP_KERNEL); if (!zip_cap) return -ENOMEM; for (i = 0; i < size; i++) { - zip_cap[i].type = zip_pre_store_caps[i]; - zip_cap[i].cap_val = hisi_qm_get_hw_info(qm, zip_basic_cap_info, - zip_pre_store_caps[i], qm->cap_ver); + zip_cap[i].type = zip_cap_query_info[i].type; + zip_cap[i].name = zip_cap_query_info[i].name; + zip_cap[i].cap_val = hisi_qm_get_cap_value(qm, zip_cap_query_info, + i, qm->cap_ver); } qm->cap_tables.dev_cap_table = zip_cap; + qm->cap_tables.dev_cap_size = size; return 0; } @@ -1230,7 +1305,7 @@ static int hisi_zip_qm_init(struct hisi_qm *qm, struct pci_dev *pdev) return ret; } - alg_msk = qm->cap_tables.dev_cap_table[ZIP_DEV_ALG_BITMAP_IDX].cap_val; + alg_msk = qm->cap_tables.dev_cap_table[ZIP_ALG_BITMAP].cap_val; ret = hisi_qm_set_algs(qm, alg_msk, zip_dev_algs, ARRAY_SIZE(zip_dev_algs)); if (ret) { pci_err(qm->pdev, "Failed to set zip algs!\n"); diff --git a/drivers/crypto/img-hash.c b/drivers/crypto/img-hash.c index 7e93159c3b6b..1dc2378aa88b 100644 --- a/drivers/crypto/img-hash.c +++ b/drivers/crypto/img-hash.c @@ -1084,7 +1084,7 @@ static const struct dev_pm_ops img_hash_pm_ops = { static struct platform_driver img_hash_driver = { .probe = img_hash_probe, - .remove_new = img_hash_remove, + .remove = img_hash_remove, .driver = { .name = "img-hash-accelerator", .pm = &img_hash_pm_ops, diff --git a/drivers/crypto/inside-secure/safexcel.c b/drivers/crypto/inside-secure/safexcel.c index f5c1912aa564..45758c7aa80e 100644 --- a/drivers/crypto/inside-secure/safexcel.c +++ b/drivers/crypto/inside-secure/safexcel.c @@ -1868,7 +1868,7 @@ MODULE_DEVICE_TABLE(of, safexcel_of_match_table); static struct platform_driver crypto_safexcel = { .probe = safexcel_probe, - .remove_new = safexcel_remove, + .remove = safexcel_remove, .driver = { .name = "crypto-safexcel", .of_match_table = safexcel_of_match_table, diff --git a/drivers/crypto/inside-secure/safexcel_hash.c b/drivers/crypto/inside-secure/safexcel_hash.c index e17577b785c3..f44c08f5f5ec 100644 --- a/drivers/crypto/inside-secure/safexcel_hash.c +++ b/drivers/crypto/inside-secure/safexcel_hash.c @@ -2093,7 +2093,7 @@ static int safexcel_xcbcmac_cra_init(struct crypto_tfm *tfm) safexcel_ahash_cra_init(tfm); ctx->aes = kmalloc(sizeof(*ctx->aes), GFP_KERNEL); - return PTR_ERR_OR_ZERO(ctx->aes); + return ctx->aes == NULL ? -ENOMEM : 0; } static void safexcel_xcbcmac_cra_exit(struct crypto_tfm *tfm) diff --git a/drivers/crypto/intel/iaa/iaa_crypto_main.c b/drivers/crypto/intel/iaa/iaa_crypto_main.c index 237f87000070..8fced88d3d06 100644 --- a/drivers/crypto/intel/iaa/iaa_crypto_main.c +++ b/drivers/crypto/intel/iaa/iaa_crypto_main.c @@ -945,12 +945,22 @@ static inline int check_completion(struct device *dev, bool only_once) { char *op_str = compress ? "compress" : "decompress"; + int status_checks = 0; int ret = 0; while (!comp->status) { if (only_once) return -EAGAIN; cpu_relax(); + if (status_checks++ >= IAA_COMPLETION_TIMEOUT) { + /* Something is wrong with the hw, disable it. */ + dev_err(dev, "%s completion timed out - " + "assuming broken hw, iaa_crypto now DISABLED\n", + op_str); + iaa_crypto_enabled = false; + ret = -ETIMEDOUT; + goto out; + } } if (comp->status != IAX_COMP_SUCCESS) { diff --git a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c index f8a77bff8844..449c6d3ab2db 100644 --- a/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c +++ b/drivers/crypto/intel/ixp4xx/ixp4xx_crypto.c @@ -1588,7 +1588,7 @@ static const struct of_device_id ixp4xx_crypto_of_match[] = { static struct platform_driver ixp_crypto_driver = { .probe = ixp_crypto_probe, - .remove_new = ixp_crypto_remove, + .remove = ixp_crypto_remove, .driver = { .name = "ixp4xx_crypto", .of_match_table = ixp4xx_crypto_of_match, diff --git a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c index 9b2d098e5eb2..8a8f6c81e010 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-aes-core.c @@ -1656,7 +1656,7 @@ list_del: /* The OCS driver is a platform device. */ static struct platform_driver kmb_ocs_aes_driver = { .probe = kmb_ocs_aes_probe, - .remove_new = kmb_ocs_aes_remove, + .remove = kmb_ocs_aes_remove, .driver = { .name = DRV_NAME, .of_match_table = kmb_ocs_aes_of_match, diff --git a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c index 5e24f2d8affc..59308926399d 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-ecc.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-ecc.c @@ -991,7 +991,7 @@ static const struct of_device_id kmb_ocs_ecc_of_match[] = { /* The OCS driver is a platform device. */ static struct platform_driver kmb_ocs_ecc_driver = { .probe = kmb_ocs_ecc_probe, - .remove_new = kmb_ocs_ecc_remove, + .remove = kmb_ocs_ecc_remove, .driver = { .name = DRV_NAME, .of_match_table = kmb_ocs_ecc_of_match, diff --git a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c index e54c79890d44..95dc8979918d 100644 --- a/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c +++ b/drivers/crypto/intel/keembay/keembay-ocs-hcu-core.c @@ -1243,7 +1243,7 @@ list_del: /* The OCS driver is a platform device. */ static struct platform_driver kmb_ocs_hcu_driver = { .probe = kmb_ocs_hcu_probe, - .remove_new = kmb_ocs_hcu_remove, + .remove = kmb_ocs_hcu_remove, .driver = { .name = DRV_NAME, .of_match_table = kmb_ocs_hcu_of_match, diff --git a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c index 78f0ea49254d..9faef33e54bd 100644 --- a/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_420xx/adf_420xx_hw_data.c @@ -375,7 +375,7 @@ static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num, else id = -EINVAL; - if (id < 0 || id > num_objs) + if (id < 0 || id >= num_objs) return NULL; return fw_objs[id]; diff --git a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c index 9fd7ec53b9f3..bbd92c017c28 100644 --- a/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c +++ b/drivers/crypto/intel/qat/qat_4xxx/adf_4xxx_hw_data.c @@ -334,7 +334,7 @@ static const char *uof_get_name(struct adf_accel_dev *accel_dev, u32 obj_num, else id = -EINVAL; - if (id < 0 || id > num_objs) + if (id < 0 || id >= num_objs) return NULL; return fw_objs[id]; diff --git a/drivers/crypto/intel/qat/qat_common/adf_aer.c b/drivers/crypto/intel/qat/qat_common/adf_aer.c index ec7913ab00a2..4cb8bd83f570 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_aer.c +++ b/drivers/crypto/intel/qat/qat_common/adf_aer.c @@ -281,8 +281,11 @@ int adf_init_aer(void) return -EFAULT; device_sriov_wq = alloc_workqueue("qat_device_sriov_wq", 0, 0); - if (!device_sriov_wq) + if (!device_sriov_wq) { + destroy_workqueue(device_reset_wq); + device_reset_wq = NULL; return -EFAULT; + } return 0; } diff --git a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h index f7ecabdf7805..eaa6388a6678 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_common_drv.h +++ b/drivers/crypto/intel/qat/qat_common/adf_common_drv.h @@ -69,7 +69,6 @@ void adf_devmgr_rm_dev(struct adf_accel_dev *accel_dev, struct adf_accel_dev *pf); struct list_head *adf_devmgr_get_head(void); struct adf_accel_dev *adf_devmgr_get_dev_by_id(u32 id); -struct adf_accel_dev *adf_devmgr_get_first(void); struct adf_accel_dev *adf_devmgr_pci_to_accel_dev(struct pci_dev *pci_dev); int adf_devmgr_verify_id(u32 id); void adf_devmgr_get_num_dev(u32 *num); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c index c42f5c25aabd..4c11ad1ebcf0 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dbgfs.c @@ -22,18 +22,13 @@ void adf_dbgfs_init(struct adf_accel_dev *accel_dev) { char name[ADF_DEVICE_NAME_LENGTH]; - void *ret; /* Create dev top level debugfs entry */ snprintf(name, sizeof(name), "%s%s_%s", ADF_DEVICE_NAME_PREFIX, accel_dev->hw_device->dev_class->name, pci_name(accel_dev->accel_pci_dev.pci_dev)); - ret = debugfs_create_dir(name, NULL); - if (IS_ERR_OR_NULL(ret)) - return; - - accel_dev->debugfs_dir = ret; + accel_dev->debugfs_dir = debugfs_create_dir(name, NULL); adf_cfg_dev_dbgfs_add(accel_dev); } @@ -59,9 +54,6 @@ EXPORT_SYMBOL_GPL(adf_dbgfs_exit); */ void adf_dbgfs_add(struct adf_accel_dev *accel_dev) { - if (!accel_dev->debugfs_dir) - return; - if (!accel_dev->is_vf) { adf_fw_counters_dbgfs_add(accel_dev); adf_heartbeat_dbgfs_add(accel_dev); @@ -77,9 +69,6 @@ void adf_dbgfs_add(struct adf_accel_dev *accel_dev) */ void adf_dbgfs_rm(struct adf_accel_dev *accel_dev) { - if (!accel_dev->debugfs_dir) - return; - if (!accel_dev->is_vf) { adf_tl_dbgfs_rm(accel_dev); adf_cnv_dbgfs_rm(accel_dev); diff --git a/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c index 96ddd1c419c4..34b9f7731c78 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c +++ b/drivers/crypto/intel/qat/qat_common/adf_dev_mgr.c @@ -276,16 +276,6 @@ unlock: } EXPORT_SYMBOL_GPL(adf_devmgr_rm_dev); -struct adf_accel_dev *adf_devmgr_get_first(void) -{ - struct adf_accel_dev *dev = NULL; - - if (!list_empty(&accel_table)) - dev = list_first_entry(&accel_table, struct adf_accel_dev, - list); - return dev; -} - /** * adf_devmgr_pci_to_accel_dev() - Get accel_dev associated with the pci_dev. * @pci_dev: Pointer to PCI device. diff --git a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c index ee0b5079de3e..2e4095c4c12c 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c +++ b/drivers/crypto/intel/qat/qat_common/adf_gen4_pm_debugfs.c @@ -42,13 +42,13 @@ struct pm_status_row { const char *key; }; -static struct pm_status_row pm_fuse_rows[] = { +static const struct pm_status_row pm_fuse_rows[] = { PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM), PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_PM_IDLE), PM_INFO_REGSET_ENTRY(fusectl0, ENABLE_DEEP_PM_IDLE), }; -static struct pm_status_row pm_info_rows[] = { +static const struct pm_status_row pm_info_rows[] = { PM_INFO_REGSET_ENTRY(pm.status, CPM_PM_STATE), PM_INFO_REGSET_ENTRY(pm.status, PENDING_WP), PM_INFO_REGSET_ENTRY(pm.status, CURRENT_WP), @@ -59,7 +59,7 @@ static struct pm_status_row pm_info_rows[] = { PM_INFO_REGSET_ENTRY(pm.main, THR_VALUE), }; -static struct pm_status_row pm_ssm_rows[] = { +static const struct pm_status_row pm_ssm_rows[] = { PM_INFO_REGSET_ENTRY(ssm.pm_enable, SSM_PM_ENABLE), PM_INFO_REGSET_ENTRY32(ssm.active_constraint, ACTIVE_CONSTRAINT), PM_INFO_REGSET_ENTRY(ssm.pm_domain_status, DOMAIN_POWER_GATED), @@ -83,7 +83,7 @@ static struct pm_status_row pm_ssm_rows[] = { PM_INFO_REGSET_ENTRY(ssm.pm_managed_status, WCP_MANAGED_COUNT), }; -static struct pm_status_row pm_log_rows[] = { +static const struct pm_status_row pm_log_rows[] = { PM_INFO_REGSET_ENTRY32(event_counters.host_msg, HOST_MSG_EVENT_COUNT), PM_INFO_REGSET_ENTRY32(event_counters.sys_pm, SYS_PM_EVENT_COUNT), PM_INFO_REGSET_ENTRY32(event_counters.local_ssm, SSM_EVENT_COUNT), @@ -91,7 +91,7 @@ static struct pm_status_row pm_log_rows[] = { PM_INFO_REGSET_ENTRY32(event_counters.unknown, UNKNOWN_EVENT_COUNT), }; -static struct pm_status_row pm_event_rows[ICP_QAT_NUMBER_OF_PM_EVENTS] = { +static const struct pm_status_row pm_event_rows[ICP_QAT_NUMBER_OF_PM_EVENTS] = { PM_INFO_REGSET_ENTRY32(event_log[0], EVENT0), PM_INFO_REGSET_ENTRY32(event_log[1], EVENT1), PM_INFO_REGSET_ENTRY32(event_log[2], EVENT2), @@ -102,14 +102,14 @@ static struct pm_status_row pm_event_rows[ICP_QAT_NUMBER_OF_PM_EVENTS] = { PM_INFO_REGSET_ENTRY32(event_log[7], EVENT7), }; -static struct pm_status_row pm_csrs_rows[] = { +static const struct pm_status_row pm_csrs_rows[] = { PM_INFO_REGSET_ENTRY32(pm.fw_init, CPM_PM_FW_INIT), PM_INFO_REGSET_ENTRY32(pm.status, CPM_PM_STATUS), PM_INFO_REGSET_ENTRY32(pm.main, CPM_PM_MASTER_FW), PM_INFO_REGSET_ENTRY32(pm.pwrreq, CPM_PM_PWRREQ), }; -static int pm_scnprint_table(char *buff, struct pm_status_row *table, +static int pm_scnprint_table(char *buff, const struct pm_status_row *table, u32 *pm_info_regs, size_t buff_size, int table_len, bool lowercase) { @@ -131,7 +131,7 @@ static int pm_scnprint_table(char *buff, struct pm_status_row *table, return wr; } -static int pm_scnprint_table_upper_keys(char *buff, struct pm_status_row *table, +static int pm_scnprint_table_upper_keys(char *buff, const struct pm_status_row *table, u32 *pm_info_regs, size_t buff_size, int table_len) { @@ -139,7 +139,7 @@ static int pm_scnprint_table_upper_keys(char *buff, struct pm_status_row *table, table_len, false); } -static int pm_scnprint_table_lower_keys(char *buff, struct pm_status_row *table, +static int pm_scnprint_table_lower_keys(char *buff, const struct pm_status_row *table, u32 *pm_info_regs, size_t buff_size, int table_len) { diff --git a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c index 65bd26b25abc..f93d9cca70ce 100644 --- a/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c +++ b/drivers/crypto/intel/qat/qat_common/adf_hw_arbiter.c @@ -90,10 +90,6 @@ void adf_exit_arb(struct adf_accel_dev *accel_dev) hw_data->get_arb_info(&info); - /* Reset arbiter configuration */ - for (i = 0; i < ADF_ARB_NUM; i++) - WRITE_CSR_ARB_SARCONFIG(csr, arb_off, i, 0); - /* Unmap worker threads to service arbiters */ for (i = 0; i < hw_data->num_engines; i++) WRITE_CSR_ARB_WT2SAM(csr, arb_off, wt_off, i, 0); diff --git a/drivers/crypto/intel/qat/qat_common/qat_hal.c b/drivers/crypto/intel/qat/qat_common/qat_hal.c index 317cafa9d11f..ef8a9cf74f0c 100644 --- a/drivers/crypto/intel/qat/qat_common/qat_hal.c +++ b/drivers/crypto/intel/qat/qat_common/qat_hal.c @@ -163,7 +163,7 @@ int qat_hal_set_ae_ctx_mode(struct icp_qat_fw_loader_handle *handle, return -EINVAL; } - /* Sets the accelaration engine context mode to either four or eight */ + /* Sets the acceleration engine context mode to either four or eight */ csr = qat_hal_rd_ae_csr(handle, ae, CTX_ENABLES); csr = IGNORE_W1C_MASK & csr; new_csr = (mode == 4) ? diff --git a/drivers/crypto/marvell/Kconfig b/drivers/crypto/marvell/Kconfig index 78217577aa54..4c25a78ab3ed 100644 --- a/drivers/crypto/marvell/Kconfig +++ b/drivers/crypto/marvell/Kconfig @@ -7,7 +7,7 @@ config CRYPTO_DEV_MARVELL config CRYPTO_DEV_MARVELL_CESA tristate "Marvell's Cryptographic Engine driver" - depends on PLAT_ORION || ARCH_MVEBU + depends on PLAT_ORION || ARCH_MVEBU || COMPILE_TEST select CRYPTO_LIB_AES select CRYPTO_LIB_DES select CRYPTO_SKCIPHER diff --git a/drivers/crypto/marvell/cesa/cesa.c b/drivers/crypto/marvell/cesa/cesa.c index 5fd31ba715c2..fa08f10e6f3f 100644 --- a/drivers/crypto/marvell/cesa/cesa.c +++ b/drivers/crypto/marvell/cesa/cesa.c @@ -375,7 +375,6 @@ static int mv_cesa_get_sram(struct platform_device *pdev, int idx) { struct mv_cesa_dev *cesa = platform_get_drvdata(pdev); struct mv_cesa_engine *engine = &cesa->engines[idx]; - const char *res_name = "sram"; struct resource *res; engine->pool = of_gen_pool_get(cesa->dev->of_node, @@ -391,19 +390,7 @@ static int mv_cesa_get_sram(struct platform_device *pdev, int idx) return -ENOMEM; } - if (cesa->caps->nengines > 1) { - if (!idx) - res_name = "sram0"; - else - res_name = "sram1"; - } - - res = platform_get_resource_byname(pdev, IORESOURCE_MEM, - res_name); - if (!res || resource_size(res) < cesa->sram_size) - return -EINVAL; - - engine->sram = devm_ioremap_resource(cesa->dev, res); + engine->sram = devm_platform_get_and_ioremap_resource(pdev, idx, &res); if (IS_ERR(engine->sram)) return PTR_ERR(engine->sram); @@ -510,25 +497,21 @@ static int mv_cesa_probe(struct platform_device *pdev) * if the clock does not exist. */ snprintf(res_name, sizeof(res_name), "cesa%u", i); - engine->clk = devm_clk_get(dev, res_name); + engine->clk = devm_clk_get_optional_enabled(dev, res_name); if (IS_ERR(engine->clk)) { - engine->clk = devm_clk_get(dev, NULL); - if (IS_ERR(engine->clk)) - engine->clk = NULL; + engine->clk = devm_clk_get_optional_enabled(dev, NULL); + if (IS_ERR(engine->clk)) { + ret = PTR_ERR(engine->clk); + goto err_cleanup; + } } snprintf(res_name, sizeof(res_name), "cesaz%u", i); - engine->zclk = devm_clk_get(dev, res_name); - if (IS_ERR(engine->zclk)) - engine->zclk = NULL; - - ret = clk_prepare_enable(engine->clk); - if (ret) - goto err_cleanup; - - ret = clk_prepare_enable(engine->zclk); - if (ret) + engine->zclk = devm_clk_get_optional_enabled(dev, res_name); + if (IS_ERR(engine->zclk)) { + ret = PTR_ERR(engine->zclk); goto err_cleanup; + } engine->regs = cesa->regs + CESA_ENGINE_OFF(i); @@ -570,13 +553,8 @@ static int mv_cesa_probe(struct platform_device *pdev) return 0; err_cleanup: - for (i = 0; i < caps->nengines; i++) { - clk_disable_unprepare(cesa->engines[i].zclk); - clk_disable_unprepare(cesa->engines[i].clk); + for (i = 0; i < caps->nengines; i++) mv_cesa_put_sram(pdev, i); - if (cesa->engines[i].irq > 0) - irq_set_affinity_hint(cesa->engines[i].irq, NULL); - } return ret; } @@ -588,12 +566,8 @@ static void mv_cesa_remove(struct platform_device *pdev) mv_cesa_remove_algs(cesa); - for (i = 0; i < cesa->caps->nengines; i++) { - clk_disable_unprepare(cesa->engines[i].zclk); - clk_disable_unprepare(cesa->engines[i].clk); + for (i = 0; i < cesa->caps->nengines; i++) mv_cesa_put_sram(pdev, i); - irq_set_affinity_hint(cesa->engines[i].irq, NULL); - } } static const struct platform_device_id mv_cesa_plat_id_table[] = { @@ -604,7 +578,7 @@ MODULE_DEVICE_TABLE(platform, mv_cesa_plat_id_table); static struct platform_driver marvell_cesa = { .probe = mv_cesa_probe, - .remove_new = mv_cesa_remove, + .remove = mv_cesa_remove, .id_table = mv_cesa_plat_id_table, .driver = { .name = "marvell-cesa", diff --git a/drivers/crypto/marvell/cesa/cipher.c b/drivers/crypto/marvell/cesa/cipher.c index 0f37dfd42d85..cf62db50f958 100644 --- a/drivers/crypto/marvell/cesa/cipher.c +++ b/drivers/crypto/marvell/cesa/cipher.c @@ -489,7 +489,7 @@ static int mv_cesa_des_op(struct skcipher_request *req, static int mv_cesa_ecb_des_encrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_ECB | @@ -500,7 +500,7 @@ static int mv_cesa_ecb_des_encrypt(struct skcipher_request *req) static int mv_cesa_ecb_des_decrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_ECB | @@ -543,7 +543,7 @@ static int mv_cesa_cbc_des_op(struct skcipher_request *req, static int mv_cesa_cbc_des_encrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_ENC); @@ -552,7 +552,7 @@ static int mv_cesa_cbc_des_encrypt(struct skcipher_request *req) static int mv_cesa_cbc_des_decrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_DEC); @@ -596,7 +596,7 @@ static int mv_cesa_des3_op(struct skcipher_request *req, static int mv_cesa_ecb_des3_ede_encrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_ECB | @@ -608,7 +608,7 @@ static int mv_cesa_ecb_des3_ede_encrypt(struct skcipher_request *req) static int mv_cesa_ecb_des3_ede_decrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_ECB | @@ -649,7 +649,7 @@ static int mv_cesa_cbc_des3_op(struct skcipher_request *req, static int mv_cesa_cbc_des3_ede_encrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC | @@ -661,7 +661,7 @@ static int mv_cesa_cbc_des3_ede_encrypt(struct skcipher_request *req) static int mv_cesa_cbc_des3_ede_decrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_CBC | @@ -725,7 +725,7 @@ static int mv_cesa_aes_op(struct skcipher_request *req, static int mv_cesa_ecb_aes_encrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_ECB | @@ -736,7 +736,7 @@ static int mv_cesa_ecb_aes_encrypt(struct skcipher_request *req) static int mv_cesa_ecb_aes_decrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_CRYPTCM_ECB | @@ -778,7 +778,7 @@ static int mv_cesa_cbc_aes_op(struct skcipher_request *req, static int mv_cesa_cbc_aes_encrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_ENC); @@ -787,7 +787,7 @@ static int mv_cesa_cbc_aes_encrypt(struct skcipher_request *req) static int mv_cesa_cbc_aes_decrypt(struct skcipher_request *req) { - struct mv_cesa_op_ctx tmpl; + struct mv_cesa_op_ctx tmpl = { }; mv_cesa_set_op_cfg(&tmpl, CESA_SA_DESC_CFG_DIR_DEC); diff --git a/drivers/crypto/mxs-dcp.c b/drivers/crypto/mxs-dcp.c index c82775dbb557..d94a26c3541a 100644 --- a/drivers/crypto/mxs-dcp.c +++ b/drivers/crypto/mxs-dcp.c @@ -225,21 +225,22 @@ static int mxs_dcp_start_dma(struct dcp_async_ctx *actx) static int mxs_dcp_run_aes(struct dcp_async_ctx *actx, struct skcipher_request *req, int init) { - dma_addr_t key_phys = 0; - dma_addr_t src_phys, dst_phys; + dma_addr_t key_phys, src_phys, dst_phys; struct dcp *sdcp = global_sdcp; struct dcp_dma_desc *desc = &sdcp->coh->desc[actx->chan]; struct dcp_aes_req_ctx *rctx = skcipher_request_ctx(req); bool key_referenced = actx->key_referenced; int ret; - if (!key_referenced) { + if (key_referenced) + key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key + AES_KEYSIZE_128, + AES_KEYSIZE_128, DMA_TO_DEVICE); + else key_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_key, 2 * AES_KEYSIZE_128, DMA_TO_DEVICE); - ret = dma_mapping_error(sdcp->dev, key_phys); - if (ret) - return ret; - } + ret = dma_mapping_error(sdcp->dev, key_phys); + if (ret) + return ret; src_phys = dma_map_single(sdcp->dev, sdcp->coh->aes_in_buf, DCP_BUF_SZ, DMA_TO_DEVICE); @@ -300,7 +301,10 @@ aes_done_run: err_dst: dma_unmap_single(sdcp->dev, src_phys, DCP_BUF_SZ, DMA_TO_DEVICE); err_src: - if (!key_referenced) + if (key_referenced) + dma_unmap_single(sdcp->dev, key_phys, AES_KEYSIZE_128, + DMA_TO_DEVICE); + else dma_unmap_single(sdcp->dev, key_phys, 2 * AES_KEYSIZE_128, DMA_TO_DEVICE); return ret; @@ -1243,7 +1247,7 @@ MODULE_DEVICE_TABLE(of, mxs_dcp_dt_ids); static struct platform_driver mxs_dcp_driver = { .probe = mxs_dcp_probe, - .remove_new = mxs_dcp_remove, + .remove = mxs_dcp_remove, .driver = { .name = "mxs-dcp", .of_match_table = mxs_dcp_dt_ids, diff --git a/drivers/crypto/n2_core.c b/drivers/crypto/n2_core.c index b11545cc5cb7..14c302d2db79 100644 --- a/drivers/crypto/n2_core.c +++ b/drivers/crypto/n2_core.c @@ -2119,7 +2119,7 @@ static struct platform_driver n2_crypto_driver = { .of_match_table = n2_crypto_match, }, .probe = n2_crypto_probe, - .remove_new = n2_crypto_remove, + .remove = n2_crypto_remove, }; static const struct of_device_id n2_mau_match[] = { @@ -2146,7 +2146,7 @@ static struct platform_driver n2_mau_driver = { .of_match_table = n2_mau_match, }, .probe = n2_mau_probe, - .remove_new = n2_mau_remove, + .remove = n2_mau_remove, }; static struct platform_driver * const drivers[] = { diff --git a/drivers/crypto/nx/nx-common-pseries.c b/drivers/crypto/nx/nx-common-pseries.c index 35f2d0d8507e..1660c5cf3641 100644 --- a/drivers/crypto/nx/nx-common-pseries.c +++ b/drivers/crypto/nx/nx-common-pseries.c @@ -133,7 +133,7 @@ struct nx842_devdata { }; static struct nx842_devdata __rcu *devdata; -static DEFINE_SPINLOCK(devdata_mutex); +static DEFINE_SPINLOCK(devdata_spinlock); #define NX842_COUNTER_INC(_x) \ static inline void nx842_inc_##_x( \ @@ -750,15 +750,15 @@ static int nx842_OF_upd(struct property *new_prop) if (!new_devdata) return -ENOMEM; - spin_lock_irqsave(&devdata_mutex, flags); + spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); + lockdep_is_held(&devdata_spinlock)); if (old_devdata) of_node = old_devdata->dev->of_node; if (!old_devdata || !of_node) { pr_err("%s: device is not available\n", __func__); - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); kfree(new_devdata); return -ENODEV; } @@ -810,7 +810,7 @@ out: old_devdata->max_sg_len); rcu_assign_pointer(devdata, new_devdata); - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); dev_set_drvdata(new_devdata->dev, new_devdata); kfree(old_devdata); @@ -821,13 +821,13 @@ error_out: dev_info(old_devdata->dev, "%s: device disabled\n", __func__); nx842_OF_set_defaults(new_devdata); rcu_assign_pointer(devdata, new_devdata); - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); dev_set_drvdata(new_devdata->dev, new_devdata); kfree(old_devdata); } else { dev_err(old_devdata->dev, "%s: could not update driver from hardware\n", __func__); - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); } if (!ret) @@ -1045,9 +1045,9 @@ static int nx842_probe(struct vio_dev *viodev, return -ENOMEM; } - spin_lock_irqsave(&devdata_mutex, flags); + spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); + lockdep_is_held(&devdata_spinlock)); if (old_devdata && old_devdata->vdev != NULL) { dev_err(&viodev->dev, "%s: Attempt to register more than one instance of the hardware\n", __func__); @@ -1062,7 +1062,7 @@ static int nx842_probe(struct vio_dev *viodev, nx842_OF_set_defaults(new_devdata); rcu_assign_pointer(devdata, new_devdata); - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); kfree(old_devdata); @@ -1101,7 +1101,7 @@ static int nx842_probe(struct vio_dev *viodev, return 0; error_unlock: - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); if (new_devdata) kfree(new_devdata->counters); kfree(new_devdata); @@ -1122,12 +1122,13 @@ static void nx842_remove(struct vio_dev *viodev) crypto_unregister_alg(&nx842_pseries_alg); - spin_lock_irqsave(&devdata_mutex, flags); - old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); of_reconfig_notifier_unregister(&nx842_of_nb); + + spin_lock_irqsave(&devdata_spinlock, flags); + old_devdata = rcu_dereference_check(devdata, + lockdep_is_held(&devdata_spinlock)); RCU_INIT_POINTER(devdata, NULL); - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); dev_set_drvdata(&viodev->dev, NULL); if (old_devdata) @@ -1257,11 +1258,11 @@ static void __exit nx842_pseries_exit(void) crypto_unregister_alg(&nx842_pseries_alg); - spin_lock_irqsave(&devdata_mutex, flags); + spin_lock_irqsave(&devdata_spinlock, flags); old_devdata = rcu_dereference_check(devdata, - lockdep_is_held(&devdata_mutex)); + lockdep_is_held(&devdata_spinlock)); RCU_INIT_POINTER(devdata, NULL); - spin_unlock_irqrestore(&devdata_mutex, flags); + spin_unlock_irqrestore(&devdata_spinlock, flags); synchronize_rcu(); if (old_devdata && old_devdata->dev) dev_set_drvdata(old_devdata->dev, NULL); diff --git a/drivers/crypto/omap-aes.c b/drivers/crypto/omap-aes.c index bad1adacbc84..e27b84616743 100644 --- a/drivers/crypto/omap-aes.c +++ b/drivers/crypto/omap-aes.c @@ -1305,7 +1305,7 @@ static SIMPLE_DEV_PM_OPS(omap_aes_pm_ops, omap_aes_suspend, omap_aes_resume); static struct platform_driver omap_aes_driver = { .probe = omap_aes_probe, - .remove_new = omap_aes_remove, + .remove = omap_aes_remove, .driver = { .name = "omap-aes", .pm = &omap_aes_pm_ops, diff --git a/drivers/crypto/omap-des.c b/drivers/crypto/omap-des.c index 209d3dc03a9b..498cbd585ed1 100644 --- a/drivers/crypto/omap-des.c +++ b/drivers/crypto/omap-des.c @@ -1115,7 +1115,7 @@ static SIMPLE_DEV_PM_OPS(omap_des_pm_ops, omap_des_suspend, omap_des_resume); static struct platform_driver omap_des_driver = { .probe = omap_des_probe, - .remove_new = omap_des_remove, + .remove = omap_des_remove, .driver = { .name = "omap-des", .pm = &omap_des_pm_ops, diff --git a/drivers/crypto/omap-sham.c b/drivers/crypto/omap-sham.c index 5bcd9ab0f72a..7021481bf027 100644 --- a/drivers/crypto/omap-sham.c +++ b/drivers/crypto/omap-sham.c @@ -2216,7 +2216,7 @@ static void omap_sham_remove(struct platform_device *pdev) static struct platform_driver omap_sham_driver = { .probe = omap_sham_probe, - .remove_new = omap_sham_remove, + .remove = omap_sham_remove, .driver = { .name = "omap-sham", .of_match_table = omap_sham_of_match, diff --git a/drivers/crypto/qce/core.c b/drivers/crypto/qce/core.c index 28b5fd823827..e228a31fe28d 100644 --- a/drivers/crypto/qce/core.c +++ b/drivers/crypto/qce/core.c @@ -299,7 +299,7 @@ MODULE_DEVICE_TABLE(of, qce_crypto_of_match); static struct platform_driver qce_crypto_driver = { .probe = qce_crypto_probe, - .remove_new = qce_crypto_remove, + .remove = qce_crypto_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = qce_crypto_of_match, diff --git a/drivers/crypto/qcom-rng.c b/drivers/crypto/qcom-rng.c index 09419e79e34c..0685ba122e8a 100644 --- a/drivers/crypto/qcom-rng.c +++ b/drivers/crypto/qcom-rng.c @@ -262,7 +262,7 @@ MODULE_DEVICE_TABLE(of, qcom_rng_of_match); static struct platform_driver qcom_rng_driver = { .probe = qcom_rng_probe, - .remove_new = qcom_rng_remove, + .remove = qcom_rng_remove, .driver = { .name = KBUILD_MODNAME, .of_match_table = of_match_ptr(qcom_rng_of_match), diff --git a/drivers/crypto/rockchip/rk3288_crypto.c b/drivers/crypto/rockchip/rk3288_crypto.c index f74b3c81ba6d..b77bdce8e7fc 100644 --- a/drivers/crypto/rockchip/rk3288_crypto.c +++ b/drivers/crypto/rockchip/rk3288_crypto.c @@ -433,7 +433,7 @@ static void rk_crypto_remove(struct platform_device *pdev) static struct platform_driver crypto_driver = { .probe = rk_crypto_probe, - .remove_new = rk_crypto_remove, + .remove = rk_crypto_remove, .driver = { .name = "rk3288-crypto", .pm = &rk_crypto_pm_ops, diff --git a/drivers/crypto/s5p-sss.c b/drivers/crypto/s5p-sss.c index 8b6e3f5c94de..57ab237e899e 100644 --- a/drivers/crypto/s5p-sss.c +++ b/drivers/crypto/s5p-sss.c @@ -2335,7 +2335,7 @@ static void s5p_aes_remove(struct platform_device *pdev) static struct platform_driver s5p_aes_crypto = { .probe = s5p_aes_probe, - .remove_new = s5p_aes_remove, + .remove = s5p_aes_remove, .driver = { .name = "s5p-secss", .of_match_table = s5p_sss_dt_match, diff --git a/drivers/crypto/sa2ul.c b/drivers/crypto/sa2ul.c index 461eca40e878..091612b066f1 100644 --- a/drivers/crypto/sa2ul.c +++ b/drivers/crypto/sa2ul.c @@ -574,7 +574,7 @@ static int sa_format_cmdl_gen(struct sa_cmdl_cfg *cfg, u8 *cmdl, /* Clear the command label */ memzero_explicit(cmdl, (SA_MAX_CMDL_WORDS * sizeof(u32))); - /* Iniialize the command update structure */ + /* Initialize the command update structure */ memzero_explicit(upd_info, sizeof(*upd_info)); if (cfg->enc_eng_id && cfg->auth_eng_id) { @@ -2489,7 +2489,7 @@ static void sa_ul_remove(struct platform_device *pdev) static struct platform_driver sa_ul_driver = { .probe = sa_ul_probe, - .remove_new = sa_ul_remove, + .remove = sa_ul_remove, .driver = { .name = "saul-crypto", .of_match_table = of_match, diff --git a/drivers/crypto/sahara.c b/drivers/crypto/sahara.c index 96d4af5d48a6..533080b0cddc 100644 --- a/drivers/crypto/sahara.c +++ b/drivers/crypto/sahara.c @@ -1421,7 +1421,7 @@ static void sahara_remove(struct platform_device *pdev) static struct platform_driver sahara_driver = { .probe = sahara_probe, - .remove_new = sahara_remove, + .remove = sahara_remove, .driver = { .name = SAHARA_NAME, .of_match_table = sahara_dt_ids, diff --git a/drivers/crypto/starfive/jh7110-cryp.c b/drivers/crypto/starfive/jh7110-cryp.c index e4dfed7ee0b0..42114e9364f0 100644 --- a/drivers/crypto/starfive/jh7110-cryp.c +++ b/drivers/crypto/starfive/jh7110-cryp.c @@ -151,7 +151,7 @@ static int starfive_cryp_probe(struct platform_device *pdev) ret = starfive_aes_register_algs(); if (ret) - goto err_algs_aes; + goto err_engine_start; ret = starfive_hash_register_algs(); if (ret) @@ -167,8 +167,6 @@ err_algs_rsa: starfive_hash_unregister_algs(); err_algs_hash: starfive_aes_unregister_algs(); -err_algs_aes: - crypto_engine_stop(cryp->engine); err_engine_start: crypto_engine_exit(cryp->engine); err_engine: @@ -193,7 +191,6 @@ static void starfive_cryp_remove(struct platform_device *pdev) starfive_hash_unregister_algs(); starfive_rsa_unregister_algs(); - crypto_engine_stop(cryp->engine); crypto_engine_exit(cryp->engine); starfive_dma_cleanup(cryp); @@ -215,7 +212,7 @@ MODULE_DEVICE_TABLE(of, starfive_dt_ids); static struct platform_driver starfive_cryp_driver = { .probe = starfive_cryp_probe, - .remove_new = starfive_cryp_remove, + .remove = starfive_cryp_remove, .driver = { .name = DRIVER_NAME, .of_match_table = starfive_dt_ids, diff --git a/drivers/crypto/starfive/jh7110-rsa.c b/drivers/crypto/starfive/jh7110-rsa.c index a778c4846025..d109c743f076 100644 --- a/drivers/crypto/starfive/jh7110-rsa.c +++ b/drivers/crypto/starfive/jh7110-rsa.c @@ -565,8 +565,6 @@ static void starfive_rsa_exit_tfm(struct crypto_akcipher *tfm) static struct akcipher_alg starfive_rsa = { .encrypt = starfive_rsa_enc, .decrypt = starfive_rsa_dec, - .sign = starfive_rsa_dec, - .verify = starfive_rsa_enc, .set_pub_key = starfive_rsa_set_pub_key, .set_priv_key = starfive_rsa_set_priv_key, .max_size = starfive_rsa_max_size, diff --git a/drivers/crypto/stm32/stm32-crc32.c b/drivers/crypto/stm32/stm32-crc32.c index e0faddbf8990..de4d0402f133 100644 --- a/drivers/crypto/stm32/stm32-crc32.c +++ b/drivers/crypto/stm32/stm32-crc32.c @@ -465,7 +465,7 @@ MODULE_DEVICE_TABLE(of, stm32_dt_ids); static struct platform_driver stm32_crc_driver = { .probe = stm32_crc_probe, - .remove_new = stm32_crc_remove, + .remove = stm32_crc_remove, .driver = { .name = DRIVER_NAME, .pm = &stm32_crc_pm_ops, diff --git a/drivers/crypto/stm32/stm32-cryp.c b/drivers/crypto/stm32/stm32-cryp.c index 937f6dab8955..14c6339c2e43 100644 --- a/drivers/crypto/stm32/stm32-cryp.c +++ b/drivers/crypto/stm32/stm32-cryp.c @@ -2771,7 +2771,7 @@ static const struct dev_pm_ops stm32_cryp_pm_ops = { static struct platform_driver stm32_cryp_driver = { .probe = stm32_cryp_probe, - .remove_new = stm32_cryp_remove, + .remove = stm32_cryp_remove, .driver = { .name = DRIVER_NAME, .pm = &stm32_cryp_pm_ops, diff --git a/drivers/crypto/stm32/stm32-hash.c b/drivers/crypto/stm32/stm32-hash.c index 351827372ea6..768b27de4737 100644 --- a/drivers/crypto/stm32/stm32-hash.c +++ b/drivers/crypto/stm32/stm32-hash.c @@ -2532,7 +2532,7 @@ static const struct dev_pm_ops stm32_hash_pm_ops = { static struct platform_driver stm32_hash_driver = { .probe = stm32_hash_probe, - .remove_new = stm32_hash_remove, + .remove = stm32_hash_remove, .driver = { .name = "stm32-hash", .pm = &stm32_hash_pm_ops, diff --git a/drivers/crypto/talitos.c b/drivers/crypto/talitos.c index 511ddcb0efd4..e8c0db687c57 100644 --- a/drivers/crypto/talitos.c +++ b/drivers/crypto/talitos.c @@ -3560,7 +3560,7 @@ static struct platform_driver talitos_driver = { .of_match_table = talitos_match, }, .probe = talitos_probe, - .remove_new = talitos_remove, + .remove = talitos_remove, }; module_platform_driver(talitos_driver); diff --git a/drivers/crypto/tegra/tegra-se-aes.c b/drivers/crypto/tegra/tegra-se-aes.c index ae7a0f8435fc..9d130592cc0a 100644 --- a/drivers/crypto/tegra/tegra-se-aes.c +++ b/drivers/crypto/tegra/tegra-se-aes.c @@ -1180,8 +1180,6 @@ static int tegra_ccm_do_one_req(struct crypto_engine *engine, void *areq) goto out; } else { rctx->cryptlen = req->cryptlen - ctx->authsize; - if (ret) - goto out; /* CTR operation */ ret = tegra_ccm_do_ctr(ctx, rctx); diff --git a/drivers/crypto/tegra/tegra-se-main.c b/drivers/crypto/tegra/tegra-se-main.c index f94c0331b148..918c0b10614d 100644 --- a/drivers/crypto/tegra/tegra-se-main.c +++ b/drivers/crypto/tegra/tegra-se-main.c @@ -312,7 +312,6 @@ static int tegra_se_probe(struct platform_device *pdev) ret = tegra_se_host1x_register(se); if (ret) { - crypto_engine_stop(se->engine); crypto_engine_exit(se->engine); return dev_err_probe(dev, ret, "failed to init host1x params\n"); } @@ -324,7 +323,6 @@ static void tegra_se_remove(struct platform_device *pdev) { struct tegra_se *se = platform_get_drvdata(pdev); - crypto_engine_stop(se->engine); crypto_engine_exit(se->engine); host1x_client_unregister(&se->client); } @@ -387,7 +385,7 @@ static struct platform_driver tegra_se_driver = { .of_match_table = tegra_se_of_match, }, .probe = tegra_se_probe, - .remove_new = tegra_se_remove, + .remove = tegra_se_remove, }; static int tegra_se_host1x_probe(struct host1x_device *dev) diff --git a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c index cb92b7fa99c6..48fee07b7e51 100644 --- a/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c +++ b/drivers/crypto/virtio/virtio_crypto_akcipher_algs.c @@ -83,23 +83,16 @@ static void virtio_crypto_dataq_akcipher_callback(struct virtio_crypto_request * case VIRTIO_CRYPTO_BADMSG: error = -EBADMSG; break; - - case VIRTIO_CRYPTO_KEY_REJECTED: - error = -EKEYREJECTED; - break; - default: error = -EIO; break; } akcipher_req = vc_akcipher_req->akcipher_req; - if (vc_akcipher_req->opcode != VIRTIO_CRYPTO_AKCIPHER_VERIFY) { - /* actuall length maybe less than dst buffer */ - akcipher_req->dst_len = len - sizeof(vc_req->status); - sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), - vc_akcipher_req->dst_buf, akcipher_req->dst_len); - } + /* actual length maybe less than dst buffer */ + akcipher_req->dst_len = len - sizeof(vc_req->status); + sg_copy_from_buffer(akcipher_req->dst, sg_nents(akcipher_req->dst), + vc_akcipher_req->dst_buf, akcipher_req->dst_len); virtio_crypto_akcipher_finalize_req(vc_akcipher_req, akcipher_req, error); } @@ -230,36 +223,27 @@ static int __virtio_crypto_akcipher_do_req(struct virtio_crypto_akcipher_request int node = dev_to_node(&vcrypto->vdev->dev); unsigned long flags; int ret; - bool verify = vc_akcipher_req->opcode == VIRTIO_CRYPTO_AKCIPHER_VERIFY; - unsigned int src_len = verify ? req->src_len + req->dst_len : req->src_len; /* out header */ sg_init_one(&outhdr_sg, req_data, sizeof(*req_data)); sgs[num_out++] = &outhdr_sg; /* src data */ - src_buf = kcalloc_node(src_len, 1, GFP_KERNEL, node); + src_buf = kcalloc_node(req->src_len, 1, GFP_KERNEL, node); if (!src_buf) return -ENOMEM; - if (verify) { - /* for verify operation, both src and dst data work as OUT direction */ - sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len); - sg_init_one(&srcdata_sg, src_buf, src_len); - sgs[num_out++] = &srcdata_sg; - } else { - sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, src_len); - sg_init_one(&srcdata_sg, src_buf, src_len); - sgs[num_out++] = &srcdata_sg; + sg_copy_to_buffer(req->src, sg_nents(req->src), src_buf, req->src_len); + sg_init_one(&srcdata_sg, src_buf, req->src_len); + sgs[num_out++] = &srcdata_sg; - /* dst data */ - dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node); - if (!dst_buf) - goto free_src; + /* dst data */ + dst_buf = kcalloc_node(req->dst_len, 1, GFP_KERNEL, node); + if (!dst_buf) + goto free_src; - sg_init_one(&dstdata_sg, dst_buf, req->dst_len); - sgs[num_out + num_in++] = &dstdata_sg; - } + sg_init_one(&dstdata_sg, dst_buf, req->dst_len); + sgs[num_out + num_in++] = &dstdata_sg; vc_akcipher_req->src_buf = src_buf; vc_akcipher_req->dst_buf = dst_buf; @@ -352,16 +336,6 @@ static int virtio_crypto_rsa_decrypt(struct akcipher_request *req) return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_DECRYPT); } -static int virtio_crypto_rsa_sign(struct akcipher_request *req) -{ - return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_SIGN); -} - -static int virtio_crypto_rsa_verify(struct akcipher_request *req) -{ - return virtio_crypto_rsa_req(req, VIRTIO_CRYPTO_AKCIPHER_VERIFY); -} - static int virtio_crypto_rsa_set_key(struct crypto_akcipher *tfm, const void *key, unsigned int keylen, @@ -524,16 +498,19 @@ static struct virtio_crypto_akcipher_algo virtio_crypto_akcipher_algs[] = { .algo.base = { .encrypt = virtio_crypto_rsa_encrypt, .decrypt = virtio_crypto_rsa_decrypt, - .sign = virtio_crypto_rsa_sign, - .verify = virtio_crypto_rsa_verify, + /* + * Must specify an arbitrary hash algorithm upon + * set_{pub,priv}_key (even though it's not used + * by encrypt/decrypt) because qemu checks for it. + */ .set_pub_key = virtio_crypto_p1pad_rsa_sha1_set_pub_key, .set_priv_key = virtio_crypto_p1pad_rsa_sha1_set_priv_key, .max_size = virtio_crypto_rsa_max_size, .init = virtio_crypto_rsa_init_tfm, .exit = virtio_crypto_rsa_exit_tfm, .base = { - .cra_name = "pkcs1pad(rsa,sha1)", - .cra_driver_name = "virtio-pkcs1-rsa-with-sha1", + .cra_name = "pkcs1pad(rsa)", + .cra_driver_name = "virtio-pkcs1-rsa", .cra_priority = 150, .cra_module = THIS_MODULE, .cra_ctxsize = sizeof(struct virtio_crypto_akcipher_ctx), diff --git a/drivers/crypto/xilinx/zynqmp-aes-gcm.c b/drivers/crypto/xilinx/zynqmp-aes-gcm.c index 7f0ec6887a39..6e72d9229410 100644 --- a/drivers/crypto/xilinx/zynqmp-aes-gcm.c +++ b/drivers/crypto/xilinx/zynqmp-aes-gcm.c @@ -438,7 +438,7 @@ MODULE_DEVICE_TABLE(of, zynqmp_aes_dt_ids); static struct platform_driver zynqmp_aes_driver = { .probe = zynqmp_aes_aead_probe, - .remove_new = zynqmp_aes_aead_remove, + .remove = zynqmp_aes_aead_remove, .driver = { .name = "zynqmp-aes", .of_match_table = zynqmp_aes_dt_ids, diff --git a/drivers/crypto/xilinx/zynqmp-sha.c b/drivers/crypto/xilinx/zynqmp-sha.c index 1bcec6f46c9c..580649f9bff8 100644 --- a/drivers/crypto/xilinx/zynqmp-sha.c +++ b/drivers/crypto/xilinx/zynqmp-sha.c @@ -248,7 +248,7 @@ static void zynqmp_sha_remove(struct platform_device *pdev) static struct platform_driver zynqmp_sha_driver = { .probe = zynqmp_sha_probe, - .remove_new = zynqmp_sha_remove, + .remove = zynqmp_sha_remove, .driver = { .name = "zynqmp-sha3-384", }, diff --git a/include/crypto/akcipher.h b/include/crypto/akcipher.h index 18a10cad07aa..cdf7da74bf2f 100644 --- a/include/crypto/akcipher.h +++ b/include/crypto/akcipher.h @@ -12,24 +12,19 @@ #include <linux/crypto.h> /** - * struct akcipher_request - public key request + * struct akcipher_request - public key cipher request * * @base: Common attributes for async crypto requests * @src: Source data - * For verify op this is signature + digest, in that case - * total size of @src is @src_len + @dst_len. - * @dst: Destination data (Should be NULL for verify op) + * @dst: Destination data * @src_len: Size of the input buffer - * For verify op it's size of signature part of @src, this part - * is supposed to be operated by cipher. - * @dst_len: Size of @dst buffer (for all ops except verify). + * @dst_len: Size of @dst buffer * It needs to be at least as big as the expected result * depending on the operation. * After operation it will be updated with the actual size of the * result. * In case of error where the dst sgl size was insufficient, * it will be updated to the size required for the operation. - * For verify op this is size of digest part in @src. * @__ctx: Start of private context data */ struct akcipher_request { @@ -55,15 +50,8 @@ struct crypto_akcipher { }; /** - * struct akcipher_alg - generic public key algorithm + * struct akcipher_alg - generic public key cipher algorithm * - * @sign: Function performs a sign operation as defined by public key - * algorithm. In case of error, where the dst_len was insufficient, - * the req->dst_len will be updated to the size required for the - * operation - * @verify: Function performs a complete verify operation as defined by - * public key algorithm, returning verification status. Requires - * digest value as input parameter. * @encrypt: Function performs an encrypt operation as defined by public key * algorithm. In case of error, where the dst_len was insufficient, * the req->dst_len will be updated to the size required for the @@ -94,8 +82,6 @@ struct crypto_akcipher { * @base: Common crypto API algorithm data structure */ struct akcipher_alg { - int (*sign)(struct akcipher_request *req); - int (*verify)(struct akcipher_request *req); int (*encrypt)(struct akcipher_request *req); int (*decrypt)(struct akcipher_request *req); int (*set_pub_key)(struct crypto_akcipher *tfm, const void *key, @@ -110,9 +96,9 @@ struct akcipher_alg { }; /** - * DOC: Generic Public Key API + * DOC: Generic Public Key Cipher API * - * The Public Key API is used with the algorithms of type + * The Public Key Cipher API is used with the algorithms of type * CRYPTO_ALG_TYPE_AKCIPHER (listed as type "akcipher" in /proc/crypto) */ @@ -243,10 +229,9 @@ static inline void akcipher_request_set_callback(struct akcipher_request *req, * * @req: public key request * @src: ptr to input scatter list - * @dst: ptr to output scatter list or NULL for verify op + * @dst: ptr to output scatter list * @src_len: size of the src input scatter list to be processed - * @dst_len: size of the dst output scatter list or size of signature - * portion in @src for verify op + * @dst_len: size of the dst output scatter list */ static inline void akcipher_request_set_crypt(struct akcipher_request *req, struct scatterlist *src, @@ -348,44 +333,6 @@ int crypto_akcipher_sync_decrypt(struct crypto_akcipher *tfm, void *dst, unsigned int dlen); /** - * crypto_akcipher_sign() - Invoke public key sign operation - * - * Function invokes the specific public key sign operation for a given - * public key algorithm - * - * @req: asymmetric key request - * - * Return: zero on success; error code in case of error - */ -static inline int crypto_akcipher_sign(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - - return crypto_akcipher_alg(tfm)->sign(req); -} - -/** - * crypto_akcipher_verify() - Invoke public key signature verification - * - * Function invokes the specific public key signature verification operation - * for a given public key algorithm. - * - * @req: asymmetric key request - * - * Note: req->dst should be NULL, req->src should point to SG of size - * (req->src_size + req->dst_size), containing signature (of req->src_size - * length) with appended digest (of req->dst_size length). - * - * Return: zero on verification success; error code in case of error. - */ -static inline int crypto_akcipher_verify(struct akcipher_request *req) -{ - struct crypto_akcipher *tfm = crypto_akcipher_reqtfm(req); - - return crypto_akcipher_alg(tfm)->verify(req); -} - -/** * crypto_akcipher_set_pub_key() - Invoke set public key operation * * Function invokes the algorithm specific set key function, which knows diff --git a/include/crypto/internal/akcipher.h b/include/crypto/internal/akcipher.h index a0fba4b2eccf..14ee62bc52b6 100644 --- a/include/crypto/internal/akcipher.h +++ b/include/crypto/internal/akcipher.h @@ -124,7 +124,7 @@ static inline struct akcipher_alg *crypto_spawn_akcipher_alg( /** * crypto_register_akcipher() -- Register public key algorithm * - * Function registers an implementation of a public key verify algorithm + * Function registers an implementation of a public key cipher algorithm * * @alg: algorithm definition * @@ -135,7 +135,7 @@ int crypto_register_akcipher(struct akcipher_alg *alg); /** * crypto_unregister_akcipher() -- Unregister public key algorithm * - * Function unregisters an implementation of a public key verify algorithm + * Function unregisters an implementation of a public key cipher algorithm * * @alg: algorithm definition */ diff --git a/include/crypto/internal/ecc.h b/include/crypto/internal/ecc.h index 065f00e4bf40..57cd75242141 100644 --- a/include/crypto/internal/ecc.h +++ b/include/crypto/internal/ecc.h @@ -42,6 +42,18 @@ #define ECC_POINT_INIT(x, y, ndigits) (struct ecc_point) { x, y, ndigits } +/* + * The integers r and s making up the signature are expected to be + * formatted as two consecutive u64 arrays of size ECC_MAX_BYTES. + * The bytes within each u64 digit are in native endianness, + * but the order of the u64 digits themselves is little endian. + * This format allows direct use by internal vli_*() functions. + */ +struct ecdsa_raw_sig { + u64 r[ECC_MAX_DIGITS]; + u64 s[ECC_MAX_DIGITS]; +}; + /** * ecc_swap_digits() - Copy ndigits from big endian array to native array * @in: Input array @@ -293,4 +305,6 @@ void ecc_point_mult_shamir(const struct ecc_point *result, const u64 *y, const struct ecc_point *q, const struct ecc_curve *curve); +extern struct crypto_template ecdsa_x962_tmpl; +extern struct crypto_template ecdsa_p1363_tmpl; #endif diff --git a/include/crypto/internal/rsa.h b/include/crypto/internal/rsa.h index e870133f4b77..071a1951b992 100644 --- a/include/crypto/internal/rsa.h +++ b/include/crypto/internal/rsa.h @@ -8,6 +8,7 @@ #ifndef _RSA_HELPER_ #define _RSA_HELPER_ #include <linux/types.h> +#include <crypto/akcipher.h> /** * rsa_key - RSA key structure @@ -53,5 +54,33 @@ int rsa_parse_pub_key(struct rsa_key *rsa_key, const void *key, int rsa_parse_priv_key(struct rsa_key *rsa_key, const void *key, unsigned int key_len); +#define RSA_PUB (true) +#define RSA_PRIV (false) + +static inline int rsa_set_key(struct crypto_akcipher *child, + unsigned int *key_size, bool is_pubkey, + const void *key, unsigned int keylen) +{ + int err; + + *key_size = 0; + + if (is_pubkey) + err = crypto_akcipher_set_pub_key(child, key, keylen); + else + err = crypto_akcipher_set_priv_key(child, key, keylen); + if (err) + return err; + + /* Find out new modulus size from rsa implementation */ + err = crypto_akcipher_maxsize(child); + if (err > PAGE_SIZE) + return -ENOTSUPP; + + *key_size = err; + return 0; +} + extern struct crypto_template rsa_pkcs1pad_tmpl; +extern struct crypto_template rsassa_pkcs1_tmpl; #endif diff --git a/include/crypto/internal/sig.h b/include/crypto/internal/sig.h index 97cb26ef8115..b16648c1a986 100644 --- a/include/crypto/internal/sig.h +++ b/include/crypto/internal/sig.h @@ -10,8 +10,88 @@ #include <crypto/algapi.h> #include <crypto/sig.h> +struct sig_instance { + void (*free)(struct sig_instance *inst); + union { + struct { + char head[offsetof(struct sig_alg, base)]; + struct crypto_instance base; + }; + struct sig_alg alg; + }; +}; + +struct crypto_sig_spawn { + struct crypto_spawn base; +}; + static inline void *crypto_sig_ctx(struct crypto_sig *tfm) { return crypto_tfm_ctx(&tfm->base); } + +/** + * crypto_register_sig() -- Register public key signature algorithm + * + * Function registers an implementation of a public key signature algorithm + * + * @alg: algorithm definition + * + * Return: zero on success; error code in case of error + */ +int crypto_register_sig(struct sig_alg *alg); + +/** + * crypto_unregister_sig() -- Unregister public key signature algorithm + * + * Function unregisters an implementation of a public key signature algorithm + * + * @alg: algorithm definition + */ +void crypto_unregister_sig(struct sig_alg *alg); + +int sig_register_instance(struct crypto_template *tmpl, + struct sig_instance *inst); + +static inline struct sig_instance *sig_instance(struct crypto_instance *inst) +{ + return container_of(&inst->alg, struct sig_instance, alg.base); +} + +static inline struct sig_instance *sig_alg_instance(struct crypto_sig *tfm) +{ + return sig_instance(crypto_tfm_alg_instance(&tfm->base)); +} + +static inline struct crypto_instance *sig_crypto_instance(struct sig_instance + *inst) +{ + return container_of(&inst->alg.base, struct crypto_instance, alg); +} + +static inline void *sig_instance_ctx(struct sig_instance *inst) +{ + return crypto_instance_ctx(sig_crypto_instance(inst)); +} + +int crypto_grab_sig(struct crypto_sig_spawn *spawn, + struct crypto_instance *inst, + const char *name, u32 type, u32 mask); + +static inline struct crypto_sig *crypto_spawn_sig(struct crypto_sig_spawn + *spawn) +{ + return crypto_spawn_tfm2(&spawn->base); +} + +static inline void crypto_drop_sig(struct crypto_sig_spawn *spawn) +{ + crypto_drop_spawn(&spawn->base); +} + +static inline struct sig_alg *crypto_spawn_sig_alg(struct crypto_sig_spawn + *spawn) +{ + return container_of(spawn->base.alg, struct sig_alg, base); +} #endif diff --git a/include/crypto/public_key.h b/include/crypto/public_key.h index b7f308977c84..81098e00c08f 100644 --- a/include/crypto/public_key.h +++ b/include/crypto/public_key.h @@ -104,9 +104,6 @@ static inline int restrict_link_by_digsig(struct key *dest_keyring, extern int query_asymmetric_key(const struct kernel_pkey_params *, struct kernel_pkey_query *); -extern int encrypt_blob(struct kernel_pkey_params *, const void *, void *); -extern int decrypt_blob(struct kernel_pkey_params *, const void *, void *); -extern int create_signature(struct kernel_pkey_params *, const void *, void *); extern int verify_signature(const struct key *, const struct public_key_signature *); diff --git a/include/crypto/sig.h b/include/crypto/sig.h index d25186bb2be3..cff41ad93824 100644 --- a/include/crypto/sig.h +++ b/include/crypto/sig.h @@ -20,6 +20,56 @@ struct crypto_sig { }; /** + * struct sig_alg - generic public key signature algorithm + * + * @sign: Function performs a sign operation as defined by public key + * algorithm. Optional. + * @verify: Function performs a complete verify operation as defined by + * public key algorithm, returning verification status. Optional. + * @set_pub_key: Function invokes the algorithm specific set public key + * function, which knows how to decode and interpret + * the BER encoded public key and parameters. Mandatory. + * @set_priv_key: Function invokes the algorithm specific set private key + * function, which knows how to decode and interpret + * the BER encoded private key and parameters. Optional. + * @key_size: Function returns key size. Mandatory. + * @digest_size: Function returns maximum digest size. Optional. + * @max_size: Function returns maximum signature size. Optional. + * @init: Initialize the cryptographic transformation object. + * This function is used to initialize the cryptographic + * transformation object. This function is called only once at + * the instantiation time, right after the transformation context + * was allocated. In case the cryptographic hardware has some + * special requirements which need to be handled by software, this + * function shall check for the precise requirement of the + * transformation and put any software fallbacks in place. + * @exit: Deinitialize the cryptographic transformation object. This is a + * counterpart to @init, used to remove various changes set in + * @init. + * + * @base: Common crypto API algorithm data structure + */ +struct sig_alg { + int (*sign)(struct crypto_sig *tfm, + const void *src, unsigned int slen, + void *dst, unsigned int dlen); + int (*verify)(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen); + int (*set_pub_key)(struct crypto_sig *tfm, + const void *key, unsigned int keylen); + int (*set_priv_key)(struct crypto_sig *tfm, + const void *key, unsigned int keylen); + unsigned int (*key_size)(struct crypto_sig *tfm); + unsigned int (*digest_size)(struct crypto_sig *tfm); + unsigned int (*max_size)(struct crypto_sig *tfm); + int (*init)(struct crypto_sig *tfm); + void (*exit)(struct crypto_sig *tfm); + + struct crypto_alg base; +}; + +/** * DOC: Generic Public Key Signature API * * The Public Key Signature API is used with the algorithms of type @@ -47,6 +97,21 @@ static inline struct crypto_tfm *crypto_sig_tfm(struct crypto_sig *tfm) return &tfm->base; } +static inline struct crypto_sig *__crypto_sig_tfm(struct crypto_tfm *tfm) +{ + return container_of(tfm, struct crypto_sig, base); +} + +static inline struct sig_alg *__crypto_sig_alg(struct crypto_alg *alg) +{ + return container_of(alg, struct sig_alg, base); +} + +static inline struct sig_alg *crypto_sig_alg(struct crypto_sig *tfm) +{ + return __crypto_sig_alg(crypto_sig_tfm(tfm)->__crt_alg); +} + /** * crypto_free_sig() - free signature tfm handle * @@ -60,16 +125,55 @@ static inline void crypto_free_sig(struct crypto_sig *tfm) } /** - * crypto_sig_maxsize() - Get len for output buffer + * crypto_sig_keysize() - Get key size + * + * Function returns the key size in bytes. + * Function assumes that the key is already set in the transformation. If this + * function is called without a setkey or with a failed setkey, you may end up + * in a NULL dereference. + * + * @tfm: signature tfm handle allocated with crypto_alloc_sig() + */ +static inline unsigned int crypto_sig_keysize(struct crypto_sig *tfm) +{ + struct sig_alg *alg = crypto_sig_alg(tfm); + + return alg->key_size(tfm); +} + +/** + * crypto_sig_digestsize() - Get maximum digest size + * + * Function returns the maximum digest size in bytes. + * Function assumes that the key is already set in the transformation. If this + * function is called without a setkey or with a failed setkey, you may end up + * in a NULL dereference. + * + * @tfm: signature tfm handle allocated with crypto_alloc_sig() + */ +static inline unsigned int crypto_sig_digestsize(struct crypto_sig *tfm) +{ + struct sig_alg *alg = crypto_sig_alg(tfm); + + return alg->digest_size(tfm); +} + +/** + * crypto_sig_maxsize() - Get maximum signature size * - * Function returns the dest buffer size required for a given key. + * Function returns the maximum signature size in bytes. * Function assumes that the key is already set in the transformation. If this - * function is called without a setkey or with a failed setkey, you will end up + * function is called without a setkey or with a failed setkey, you may end up * in a NULL dereference. * * @tfm: signature tfm handle allocated with crypto_alloc_sig() */ -int crypto_sig_maxsize(struct crypto_sig *tfm); +static inline unsigned int crypto_sig_maxsize(struct crypto_sig *tfm) +{ + struct sig_alg *alg = crypto_sig_alg(tfm); + + return alg->max_size(tfm); +} /** * crypto_sig_sign() - Invoke signing operation @@ -84,9 +188,14 @@ int crypto_sig_maxsize(struct crypto_sig *tfm); * * Return: zero on success; error code in case of error */ -int crypto_sig_sign(struct crypto_sig *tfm, - const void *src, unsigned int slen, - void *dst, unsigned int dlen); +static inline int crypto_sig_sign(struct crypto_sig *tfm, + const void *src, unsigned int slen, + void *dst, unsigned int dlen) +{ + struct sig_alg *alg = crypto_sig_alg(tfm); + + return alg->sign(tfm, src, slen, dst, dlen); +} /** * crypto_sig_verify() - Invoke signature verification @@ -102,9 +211,14 @@ int crypto_sig_sign(struct crypto_sig *tfm, * * Return: zero on verification success; error code in case of error. */ -int crypto_sig_verify(struct crypto_sig *tfm, - const void *src, unsigned int slen, - const void *digest, unsigned int dlen); +static inline int crypto_sig_verify(struct crypto_sig *tfm, + const void *src, unsigned int slen, + const void *digest, unsigned int dlen) +{ + struct sig_alg *alg = crypto_sig_alg(tfm); + + return alg->verify(tfm, src, slen, digest, dlen); +} /** * crypto_sig_set_pubkey() - Invoke set public key operation @@ -119,8 +233,13 @@ int crypto_sig_verify(struct crypto_sig *tfm, * * Return: zero on success; error code in case of error */ -int crypto_sig_set_pubkey(struct crypto_sig *tfm, - const void *key, unsigned int keylen); +static inline int crypto_sig_set_pubkey(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct sig_alg *alg = crypto_sig_alg(tfm); + + return alg->set_pub_key(tfm, key, keylen); +} /** * crypto_sig_set_privkey() - Invoke set private key operation @@ -135,6 +254,11 @@ int crypto_sig_set_pubkey(struct crypto_sig *tfm, * * Return: zero on success; error code in case of error */ -int crypto_sig_set_privkey(struct crypto_sig *tfm, - const void *key, unsigned int keylen); +static inline int crypto_sig_set_privkey(struct crypto_sig *tfm, + const void *key, unsigned int keylen) +{ + struct sig_alg *alg = crypto_sig_alg(tfm); + + return alg->set_priv_key(tfm, key, keylen); +} #endif diff --git a/include/linux/asn1_decoder.h b/include/linux/asn1_decoder.h index 83f9c6e1e5e9..b41bce82a191 100644 --- a/include/linux/asn1_decoder.h +++ b/include/linux/asn1_decoder.h @@ -9,6 +9,7 @@ #define _LINUX_ASN1_DECODER_H #include <linux/asn1.h> +#include <linux/types.h> struct asn1_decoder; diff --git a/include/linux/asn1_encoder.h b/include/linux/asn1_encoder.h index 08cd0c2ad34f..d17484dffb74 100644 --- a/include/linux/asn1_encoder.h +++ b/include/linux/asn1_encoder.h @@ -6,7 +6,6 @@ #include <linux/types.h> #include <linux/asn1.h> #include <linux/asn1_ber_bytecode.h> -#include <linux/bug.h> #define asn1_oid_len(oid) (sizeof(oid)/sizeof(u32)) unsigned char * diff --git a/include/linux/hisi_acc_qm.h b/include/linux/hisi_acc_qm.h index 9d7754ad5e9b..6dbd0d49628f 100644 --- a/include/linux/hisi_acc_qm.h +++ b/include/linux/hisi_acc_qm.h @@ -229,6 +229,12 @@ struct hisi_qm_status { struct hisi_qm; +enum acc_err_result { + ACC_ERR_NONE, + ACC_ERR_NEED_RESET, + ACC_ERR_RECOVERED, +}; + struct hisi_qm_err_info { char *acpi_rst; u32 msi_wr_port; @@ -257,9 +263,9 @@ struct hisi_qm_err_ini { void (*close_axi_master_ooo)(struct hisi_qm *qm); void (*open_sva_prefetch)(struct hisi_qm *qm); void (*close_sva_prefetch)(struct hisi_qm *qm); - void (*log_dev_hw_err)(struct hisi_qm *qm, u32 err_sts); void (*show_last_dfx_regs)(struct hisi_qm *qm); void (*err_info_init)(struct hisi_qm *qm); + enum acc_err_result (*get_err_result)(struct hisi_qm *qm); }; struct hisi_qm_cap_info { @@ -274,13 +280,25 @@ struct hisi_qm_cap_info { u32 v3_val; }; +struct hisi_qm_cap_query_info { + u32 type; + const char *name; + u32 offset; + u32 v1_val; + u32 v2_val; + u32 v3_val; +}; + struct hisi_qm_cap_record { u32 type; + const char *name; u32 cap_val; }; struct hisi_qm_cap_tables { + u32 qm_cap_size; struct hisi_qm_cap_record *qm_cap_table; + u32 dev_cap_size; struct hisi_qm_cap_record *dev_cap_table; }; @@ -436,37 +454,6 @@ struct hisi_qp { struct uacce_queue *uacce_q; }; -static inline int q_num_set(const char *val, const struct kernel_param *kp, - unsigned int device) -{ - struct pci_dev *pdev; - u32 n, q_num; - int ret; - - if (!val) - return -EINVAL; - - pdev = pci_get_device(PCI_VENDOR_ID_HUAWEI, device, NULL); - if (!pdev) { - q_num = min_t(u32, QM_QNUM_V1, QM_QNUM_V2); - pr_info("No device found currently, suppose queue number is %u\n", - q_num); - } else { - if (pdev->revision == QM_HW_V1) - q_num = QM_QNUM_V1; - else - q_num = QM_QNUM_V2; - - pci_dev_put(pdev); - } - - ret = kstrtou32(val, 10, &n); - if (ret || n < QM_MIN_QNUM || n > q_num) - return -EINVAL; - - return param_set_int(val, kp); -} - static inline int vfs_num_set(const char *val, const struct kernel_param *kp) { u32 n; @@ -526,6 +513,8 @@ static inline void hisi_qm_del_list(struct hisi_qm *qm, struct hisi_qm_list *qm_ mutex_unlock(&qm_list->lock); } +int hisi_qm_q_num_set(const char *val, const struct kernel_param *kp, + unsigned int device); int hisi_qm_init(struct hisi_qm *qm); void hisi_qm_uninit(struct hisi_qm *qm); int hisi_qm_start(struct hisi_qm *qm); @@ -583,6 +572,9 @@ void hisi_qm_regs_dump(struct seq_file *s, struct debugfs_regset32 *regset); u32 hisi_qm_get_hw_info(struct hisi_qm *qm, const struct hisi_qm_cap_info *info_table, u32 index, bool is_read); +u32 hisi_qm_get_cap_value(struct hisi_qm *qm, + const struct hisi_qm_cap_query_info *info_table, + u32 index, bool is_read); int hisi_qm_set_algs(struct hisi_qm *qm, u64 alg_msk, const struct qm_dev_alg *dev_algs, u32 dev_algs_size); diff --git a/include/linux/slab.h b/include/linux/slab.h index b35e2db7eb0e..0268ea7abf8b 100644 --- a/include/linux/slab.h +++ b/include/linux/slab.h @@ -448,6 +448,7 @@ void kfree_sensitive(const void *objp); size_t __ksize(const void *objp); DEFINE_FREE(kfree, void *, if (!IS_ERR_OR_NULL(_T)) kfree(_T)) +DEFINE_FREE(kfree_sensitive, void *, if (_T) kfree_sensitive(_T)) /** * ksize - Report actual allocation size of associated object diff --git a/include/uapi/linux/cryptouser.h b/include/uapi/linux/cryptouser.h index 20a6c0fc149e..db05e0419972 100644 --- a/include/uapi/linux/cryptouser.h +++ b/include/uapi/linux/cryptouser.h @@ -64,6 +64,7 @@ enum crypto_attr_type_t { CRYPTOCFGA_STAT_AKCIPHER, /* No longer supported, do not use. */ CRYPTOCFGA_STAT_KPP, /* No longer supported, do not use. */ CRYPTOCFGA_STAT_ACOMP, /* No longer supported, do not use. */ + CRYPTOCFGA_REPORT_SIG, /* struct crypto_report_sig */ __CRYPTOCFGA_MAX #define CRYPTOCFGA_MAX (__CRYPTOCFGA_MAX - 1) @@ -207,6 +208,10 @@ struct crypto_report_acomp { char type[CRYPTO_MAX_NAME]; }; +struct crypto_report_sig { + char type[CRYPTO_MAX_NAME]; +}; + #define CRYPTO_REPORT_MAXSIZE (sizeof(struct crypto_user_alg) + \ sizeof(struct crypto_report_blkcipher)) diff --git a/include/uapi/linux/virtio_crypto.h b/include/uapi/linux/virtio_crypto.h index 71a54a6849ca..2fccb64c9d6b 100644 --- a/include/uapi/linux/virtio_crypto.h +++ b/include/uapi/linux/virtio_crypto.h @@ -329,6 +329,7 @@ struct virtio_crypto_op_header { VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x00) #define VIRTIO_CRYPTO_AKCIPHER_DECRYPT \ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x01) + /* akcipher sign/verify opcodes are deprecated */ #define VIRTIO_CRYPTO_AKCIPHER_SIGN \ VIRTIO_CRYPTO_OPCODE(VIRTIO_CRYPTO_SERVICE_AKCIPHER, 0x02) #define VIRTIO_CRYPTO_AKCIPHER_VERIFY \ diff --git a/kernel/padata.c b/kernel/padata.c index d899f34558af..d51bbc76b227 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -521,13 +521,6 @@ void __init padata_do_multithreaded(struct padata_mt_job *job) ps.chunk_size = max(ps.chunk_size, 1ul); ps.chunk_size = roundup(ps.chunk_size, job->align); - /* - * chunk_size can be 0 if the caller sets min_chunk to 0. So force it - * to at least 1 to prevent divide-by-0 panic in padata_mt_helper().` - */ - if (!ps.chunk_size) - ps.chunk_size = 1U; - list_for_each_entry(pw, &works, pw_list) if (job->numa_aware) { int old_node = atomic_read(&last_used_nid); diff --git a/lib/crc32.c b/lib/crc32.c index 5649847d0a8d..ff587fee3893 100644 --- a/lib/crc32.c +++ b/lib/crc32.c @@ -205,7 +205,11 @@ EXPORT_SYMBOL(crc32_le); EXPORT_SYMBOL(__crc32c_le); u32 __pure crc32_le_base(u32, unsigned char const *, size_t) __alias(crc32_le); +EXPORT_SYMBOL(crc32_le_base); + u32 __pure __crc32c_le_base(u32, unsigned char const *, size_t) __alias(__crc32c_le); +EXPORT_SYMBOL(__crc32c_le_base); + u32 __pure crc32_be_base(u32, unsigned char const *, size_t) __alias(crc32_be); /* diff --git a/lib/crypto/Makefile b/lib/crypto/Makefile index 969baab8c805..01fac1cd05a1 100644 --- a/lib/crypto/Makefile +++ b/lib/crypto/Makefile @@ -58,3 +58,5 @@ libcurve25519-y += curve25519-selftest.o endif obj-$(CONFIG_MPILIB) += mpi/ + +obj-$(CONFIG_CRYPTO_MANAGER_EXTRA_TESTS) += simd.o diff --git a/lib/crypto/mpi/mpi-bit.c b/lib/crypto/mpi/mpi-bit.c index 835a2f0622a0..934d81311360 100644 --- a/lib/crypto/mpi/mpi-bit.c +++ b/lib/crypto/mpi/mpi-bit.c @@ -95,6 +95,7 @@ int mpi_set_bit(MPI a, unsigned int n) a->d[limbno] |= (A_LIMB_1<<bitno); return 0; } +EXPORT_SYMBOL_GPL(mpi_set_bit); /* * Shift A by N bits to the right. diff --git a/lib/crypto/simd.c b/lib/crypto/simd.c new file mode 100644 index 000000000000..9c36cb3bb49c --- /dev/null +++ b/lib/crypto/simd.c @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +/* + * SIMD testing utility functions + * + * Copyright 2024 Google LLC + */ + +#include <crypto/internal/simd.h> + +DEFINE_PER_CPU(bool, crypto_simd_disabled_for_test); +EXPORT_PER_CPU_SYMBOL_GPL(crypto_simd_disabled_for_test); diff --git a/security/integrity/ima/ima_main.c b/security/integrity/ima/ima_main.c index eb153a1855c4..9b87556b03a7 100644 --- a/security/integrity/ima/ima_main.c +++ b/security/integrity/ima/ima_main.c @@ -1111,7 +1111,7 @@ EXPORT_SYMBOL_GPL(ima_measure_critical_data); #ifdef CONFIG_INTEGRITY_ASYMMETRIC_KEYS /** - * ima_kernel_module_request - Prevent crypto-pkcs1pad(rsa,*) requests + * ima_kernel_module_request - Prevent crypto-pkcs1(rsa,*) requests * @kmod_name: kernel module name * * Avoid a verification loop where verifying the signature of the modprobe @@ -1125,7 +1125,7 @@ EXPORT_SYMBOL_GPL(ima_measure_critical_data); * algorithm on the fly, but crypto_larval_lookup() will try to use alg_name * in order to load a kernel module with same name. * - * Since we don't have any real "crypto-pkcs1pad(rsa,*)" kernel modules, + * Since we don't have any real "crypto-pkcs1(rsa,*)" kernel modules, * we are safe to fail such module request from crypto_larval_lookup(), and * avoid the verification loop. * @@ -1133,7 +1133,7 @@ EXPORT_SYMBOL_GPL(ima_measure_critical_data); */ static int ima_kernel_module_request(char *kmod_name) { - if (strncmp(kmod_name, "crypto-pkcs1pad(rsa,", 20) == 0) + if (strncmp(kmod_name, "crypto-pkcs1(rsa,", 17) == 0) return -EINVAL; return 0; |