arch/arm64/crypto/crc32-ce-glue.c

   1 /*
   2  * Accelerated CRC32(C) using arm64 NEON and Crypto Extensions instructions
   3  *
   4  * Copyright (C) 2016 Linaro Ltd <ard.biesheuvel@linaro.org>
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  */
  10
  11 #include <linux/cpufeature.h>
  12 #include <linux/crc32.h>
  13 #include <linux/init.h>
  14 #include <linux/kernel.h>
  15 #include <linux/module.h>
  16 #include <linux/string.h>
  17
  18 #include <crypto/internal/hash.h>
  19
  20 #include <asm/hwcap.h>
  21 #include <asm/neon.h>
  22 #include <asm/unaligned.h>
  23
  24 #define PMULL_MIN_LEN           64L     /* minimum size of buffer
  25                                          * for crc32_pmull_le_16 */
  26 #define SCALE_F                 16L     /* size of NEON register */
  27
  28 asmlinkage u32 crc32_pmull_le(const u8 buf[], u64 len, u32 init_crc);
  29 asmlinkage u32 crc32_armv8_le(u32 init_crc, const u8 buf[], size_t len);
  30
  31 asmlinkage u32 crc32c_pmull_le(const u8 buf[], u64 len, u32 init_crc);
  32 asmlinkage u32 crc32c_armv8_le(u32 init_crc, const u8 buf[], size_t len);
  33
  34 static u32 (*fallback_crc32)(u32 init_crc, const u8 buf[], size_t len);
  35 static u32 (*fallback_crc32c)(u32 init_crc, const u8 buf[], size_t len);
  36
  37 static int crc32_pmull_cra_init(struct crypto_tfm *tfm)
  38 {
  39         u32 *key = crypto_tfm_ctx(tfm);
  40
  41         *key = 0;
  42         return 0;
  43 }
  44
  45 static int crc32c_pmull_cra_init(struct crypto_tfm *tfm)
  46 {
  47         u32 *key = crypto_tfm_ctx(tfm);
  48
  49         *key = ~0;
  50         return 0;
  51 }
  52
  53 static int crc32_pmull_setkey(struct crypto_shash *hash, const u8 *key,
  54                               unsigned int keylen)
  55 {
  56         u32 *mctx = crypto_shash_ctx(hash);
  57
  58         if (keylen != sizeof(u32)) {
  59                 crypto_shash_set_flags(hash, CRYPTO_TFM_RES_BAD_KEY_LEN);
  60                 return -EINVAL;
  61         }
  62         *mctx = le32_to_cpup((__le32 *)key);
  63         return 0;
  64 }
  65
  66 static int crc32_pmull_init(struct shash_desc *desc)
  67 {
  68         u32 *mctx = crypto_shash_ctx(desc->tfm);
  69         u32 *crc = shash_desc_ctx(desc);
  70
  71         *crc = *mctx;
  72         return 0;
  73 }
  74
  75 static int crc32_update(struct shash_desc *desc, const u8 *data,
  76                         unsigned int length)
  77 {
  78         u32 *crc = shash_desc_ctx(desc);
  79
  80         *crc = crc32_armv8_le(*crc, data, length);
  81         return 0;
  82 }
  83
  84 static int crc32c_update(struct shash_desc *desc, const u8 *data,
  85                          unsigned int length)
  86 {
  87         u32 *crc = shash_desc_ctx(desc);
  88
  89         *crc = crc32c_armv8_le(*crc, data, length);
  90         return 0;
  91 }
  92
  93 static int crc32_pmull_update(struct shash_desc *desc, const u8 *data,
  94                          unsigned int length)
  95 {
  96         u32 *crc = shash_desc_ctx(desc);
  97         unsigned int l;
  98
  99         if ((u64)data % SCALE_F) {
 100                 l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
 101
 102                 *crc = fallback_crc32(*crc, data, l);
 103
 104                 data += l;
 105                 length -= l;
 106         }
 107
 108         if (length >= PMULL_MIN_LEN) {
 109                 l = round_down(length, SCALE_F);
 110
 111                 kernel_neon_begin_partial(10);
 112                 *crc = crc32_pmull_le(data, l, *crc);
 113                 kernel_neon_end();
 114
 115                 data += l;
 116                 length -= l;
 117         }
 118
 119         if (length > 0)
 120                 *crc = fallback_crc32(*crc, data, length);
 121
 122         return 0;
 123 }
 124
 125 static int crc32c_pmull_update(struct shash_desc *desc, const u8 *data,
 126                          unsigned int length)
 127 {
 128         u32 *crc = shash_desc_ctx(desc);
 129         unsigned int l;
 130
 131         if ((u64)data % SCALE_F) {
 132                 l = min_t(u32, length, SCALE_F - ((u64)data % SCALE_F));
 133
 134                 *crc = fallback_crc32c(*crc, data, l);
 135
 136                 data += l;
 137                 length -= l;
 138         }
 139
 140         if (length >= PMULL_MIN_LEN) {
 141                 l = round_down(length, SCALE_F);
 142
 143                 kernel_neon_begin_partial(10);
 144                 *crc = crc32c_pmull_le(data, l, *crc);
 145                 kernel_neon_end();
 146
 147                 data += l;
 148                 length -= l;
 149         }
 150
 151         if (length > 0) {
 152                 *crc = fallback_crc32c(*crc, data, length);
 153         }
 154
 155         return 0;
 156 }
 157
 158 static int crc32_pmull_final(struct shash_desc *desc, u8 *out)
 159 {
 160         u32 *crc = shash_desc_ctx(desc);
 161
 162         put_unaligned_le32(*crc, out);
 163         return 0;
 164 }
 165
 166 static int crc32c_pmull_final(struct shash_desc *desc, u8 *out)
 167 {
 168         u32 *crc = shash_desc_ctx(desc);
 169
 170         put_unaligned_le32(~*crc, out);
 171         return 0;
 172 }
 173
 174 static struct shash_alg crc32_pmull_algs[] = { {
 175         .setkey                 = crc32_pmull_setkey,
 176         .init                   = crc32_pmull_init,
 177         .update                 = crc32_update,
 178         .final                  = crc32_pmull_final,
 179         .descsize               = sizeof(u32),
 180         .digestsize             = sizeof(u32),
 181
 182         .base.cra_ctxsize       = sizeof(u32),
 183         .base.cra_init          = crc32_pmull_cra_init,
 184         .base.cra_name          = "crc32",
 185         .base.cra_driver_name   = "crc32-arm64-ce",
 186         .base.cra_priority      = 200,
 187         .base.cra_blocksize     = 1,
 188         .base.cra_module        = THIS_MODULE,
 189 }, {
 190         .setkey                 = crc32_pmull_setkey,
 191         .init                   = crc32_pmull_init,
 192         .update                 = crc32c_update,
 193         .final                  = crc32c_pmull_final,
 194         .descsize               = sizeof(u32),
 195         .digestsize             = sizeof(u32),
 196
 197         .base.cra_ctxsize       = sizeof(u32),
 198         .base.cra_init          = crc32c_pmull_cra_init,
 199         .base.cra_name          = "crc32c",
 200         .base.cra_driver_name   = "crc32c-arm64-ce",
 201         .base.cra_priority      = 200,
 202         .base.cra_blocksize     = 1,
 203         .base.cra_module        = THIS_MODULE,
 204 } };
 205
 206 static int __init crc32_pmull_mod_init(void)
 207 {
 208         if (IS_ENABLED(CONFIG_KERNEL_MODE_NEON) && (elf_hwcap & HWCAP_PMULL)) {
 209                 crc32_pmull_algs[0].update = crc32_pmull_update;
 210                 crc32_pmull_algs[1].update = crc32c_pmull_update;
 211
 212                 if (elf_hwcap & HWCAP_CRC32) {
 213                         fallback_crc32 = crc32_armv8_le;
 214                         fallback_crc32c = crc32c_armv8_le;
 215                 } else {
 216                         fallback_crc32 = crc32_le;
 217                         fallback_crc32c = __crc32c_le;
 218                 }
 219         } else if (!(elf_hwcap & HWCAP_CRC32)) {
 220                 return -ENODEV;
 221         }
 222         return crypto_register_shashes(crc32_pmull_algs,
 223                                        ARRAY_SIZE(crc32_pmull_algs));
 224 }
 225
 226 static void __exit crc32_pmull_mod_exit(void)
 227 {
 228         crypto_unregister_shashes(crc32_pmull_algs,
 229                                   ARRAY_SIZE(crc32_pmull_algs));
 230 }
 231
 232 static const struct cpu_feature crc32_cpu_feature[] = {
 233         { cpu_feature(CRC32) }, { cpu_feature(PMULL) }, { }
 234 };
 235 MODULE_DEVICE_TABLE(cpu, crc32_cpu_feature);
 236
 237 module_init(crc32_pmull_mod_init);
 238 module_exit(crc32_pmull_mod_exit);
 239
 240 MODULE_AUTHOR("Ard Biesheuvel <ard.biesheuvel@linaro.org>");
 241 MODULE_LICENSE("GPL v2");