crypto/gcm.c

   1 /*
   2  * GCM: Galois/Counter Mode.
   3  *
   4  * Copyright (c) 2007 Nokia Siemens Networks - Mikko Herranen <mh1@iki.fi>
   5  *
   6  * This program is free software; you can redistribute it and/or modify it
   7  * under the terms of the GNU General Public License version 2 as published
   8  * by the Free Software Foundation.
   9  */
  10
  11 #include <crypto/gf128mul.h>
  12 #include <crypto/internal/aead.h>
  13 #include <crypto/internal/skcipher.h>
  14 #include <crypto/internal/hash.h>
  15 #include <crypto/scatterwalk.h>
  16 #include <crypto/hash.h>
  17 #include "internal.h"
  18 #include <linux/completion.h>
  19 #include <linux/err.h>
  20 #include <linux/init.h>
  21 #include <linux/kernel.h>
  22 #include <linux/module.h>
  23 #include <linux/slab.h>
  24
  25 struct gcm_instance_ctx {
  26         struct crypto_skcipher_spawn ctr;
  27         struct crypto_ahash_spawn ghash;
  28 };
  29
  30 struct crypto_gcm_ctx {
  31         struct crypto_ablkcipher *ctr;
  32         struct crypto_ahash *ghash;
  33 };
  34
  35 struct crypto_rfc4106_ctx {
  36         struct crypto_aead *child;
  37         u8 nonce[4];
  38 };
  39
  40 struct crypto_gcm_ghash_ctx {
  41         unsigned int cryptlen;
  42         struct scatterlist *src;
  43         void (*complete)(struct aead_request *req, int err);
  44 };
  45
  46 struct crypto_gcm_req_priv_ctx {
  47         u8 auth_tag[16];
  48         u8 iauth_tag[16];
  49         struct scatterlist src[2];
  50         struct scatterlist dst[2];
  51         struct crypto_gcm_ghash_ctx ghash_ctx;
  52         union {
  53                 struct ahash_request ahreq;
  54                 struct ablkcipher_request abreq;
  55         } u;
  56 };
  57
  58 struct crypto_gcm_setkey_result {
  59         int err;
  60         struct completion completion;
  61 };
  62
  63 static void *gcm_zeroes;
  64
  65 static inline struct crypto_gcm_req_priv_ctx *crypto_gcm_reqctx(
  66         struct aead_request *req)
  67 {
  68         unsigned long align = crypto_aead_alignmask(crypto_aead_reqtfm(req));
  69
  70         return (void *)PTR_ALIGN((u8 *)aead_request_ctx(req), align + 1);
  71 }
  72
  73 static void crypto_gcm_setkey_done(struct crypto_async_request *req, int err)
  74 {
  75         struct crypto_gcm_setkey_result *result = req->data;
  76
  77         if (err == -EINPROGRESS)
  78                 return;
  79
  80         result->err = err;
  81         complete(&result->completion);
  82 }
  83
  84 static int crypto_gcm_setkey(struct crypto_aead *aead, const u8 *key,
  85                              unsigned int keylen)
  86 {
  87         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
  88         struct crypto_ahash *ghash = ctx->ghash;
  89         struct crypto_ablkcipher *ctr = ctx->ctr;
  90         struct {
  91                 be128 hash;
  92                 u8 iv[8];
  93
  94                 struct crypto_gcm_setkey_result result;
  95
  96                 struct scatterlist sg[1];
  97                 struct ablkcipher_request req;
  98         } *data;
  99         int err;
 100
 101         crypto_ablkcipher_clear_flags(ctr, CRYPTO_TFM_REQ_MASK);
 102         crypto_ablkcipher_set_flags(ctr, crypto_aead_get_flags(aead) &
 103                                    CRYPTO_TFM_REQ_MASK);
 104
 105         err = crypto_ablkcipher_setkey(ctr, key, keylen);
 106         if (err)
 107                 return err;
 108
 109         crypto_aead_set_flags(aead, crypto_ablkcipher_get_flags(ctr) &
 110                                        CRYPTO_TFM_RES_MASK);
 111
 112         data = kzalloc(sizeof(*data) + crypto_ablkcipher_reqsize(ctr),
 113                        GFP_KERNEL);
 114         if (!data)
 115                 return -ENOMEM;
 116
 117         init_completion(&data->result.completion);
 118         sg_init_one(data->sg, &data->hash, sizeof(data->hash));
 119         ablkcipher_request_set_tfm(&data->req, ctr);
 120         ablkcipher_request_set_callback(&data->req, CRYPTO_TFM_REQ_MAY_SLEEP |
 121                                                     CRYPTO_TFM_REQ_MAY_BACKLOG,
 122                                         crypto_gcm_setkey_done,
 123                                         &data->result);
 124         ablkcipher_request_set_crypt(&data->req, data->sg, data->sg,
 125                                      sizeof(data->hash), data->iv);
 126
 127         err = crypto_ablkcipher_encrypt(&data->req);
 128         if (err == -EINPROGRESS || err == -EBUSY) {
 129                 err = wait_for_completion_interruptible(
 130                         &data->result.completion);
 131                 if (!err)
 132                         err = data->result.err;
 133         }
 134
 135         if (err)
 136                 goto out;
 137
 138         crypto_ahash_clear_flags(ghash, CRYPTO_TFM_REQ_MASK);
 139         crypto_ahash_set_flags(ghash, crypto_aead_get_flags(aead) &
 140                                CRYPTO_TFM_REQ_MASK);
 141         err = crypto_ahash_setkey(ghash, (u8 *)&data->hash, sizeof(be128));
 142         crypto_aead_set_flags(aead, crypto_ahash_get_flags(ghash) &
 143                               CRYPTO_TFM_RES_MASK);
 144
 145 out:
 146         kfree(data);
 147         return err;
 148 }
 149
 150 static int crypto_gcm_setauthsize(struct crypto_aead *tfm,
 151                                   unsigned int authsize)
 152 {
 153         switch (authsize) {
 154         case 4:
 155         case 8:
 156         case 12:
 157         case 13:
 158         case 14:
 159         case 15:
 160         case 16:
 161                 break;
 162         default:
 163                 return -EINVAL;
 164         }
 165
 166         return 0;
 167 }
 168
 169 static void crypto_gcm_init_crypt(struct ablkcipher_request *ablk_req,
 170                                   struct aead_request *req,
 171                                   unsigned int cryptlen)
 172 {
 173         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 174         struct crypto_gcm_ctx *ctx = crypto_aead_ctx(aead);
 175         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 176         struct scatterlist *dst;
 177         __be32 counter = cpu_to_be32(1);
 178
 179         memset(pctx->auth_tag, 0, sizeof(pctx->auth_tag));
 180         memcpy(req->iv + 12, &counter, 4);
 181
 182         sg_init_table(pctx->src, 2);
 183         sg_set_buf(pctx->src, pctx->auth_tag, sizeof(pctx->auth_tag));
 184         scatterwalk_sg_chain(pctx->src, 2, req->src);
 185
 186         dst = pctx->src;
 187         if (req->src != req->dst) {
 188                 sg_init_table(pctx->dst, 2);
 189                 sg_set_buf(pctx->dst, pctx->auth_tag, sizeof(pctx->auth_tag));
 190                 scatterwalk_sg_chain(pctx->dst, 2, req->dst);
 191                 dst = pctx->dst;
 192         }
 193
 194         ablkcipher_request_set_tfm(ablk_req, ctx->ctr);
 195         ablkcipher_request_set_crypt(ablk_req, pctx->src, dst,
 196                                      cryptlen + sizeof(pctx->auth_tag),
 197                                      req->iv);
 198 }
 199
 200 static inline unsigned int gcm_remain(unsigned int len)
 201 {
 202         len &= 0xfU;
 203         return len ? 16 - len : 0;
 204 }
 205
 206 static void gcm_hash_len_done(struct crypto_async_request *areq, int err);
 207 static void gcm_hash_final_done(struct crypto_async_request *areq, int err);
 208
 209 static int gcm_hash_update(struct aead_request *req,
 210                            struct crypto_gcm_req_priv_ctx *pctx,
 211                            crypto_completion_t complete,
 212                            struct scatterlist *src,
 213                            unsigned int len)
 214 {
 215         struct ahash_request *ahreq = &pctx->u.ahreq;
 216
 217         ahash_request_set_callback(ahreq, aead_request_flags(req),
 218                                    complete, req);
 219         ahash_request_set_crypt(ahreq, src, NULL, len);
 220
 221         return crypto_ahash_update(ahreq);
 222 }
 223
 224 static int gcm_hash_remain(struct aead_request *req,
 225                            struct crypto_gcm_req_priv_ctx *pctx,
 226                            unsigned int remain,
 227                            crypto_completion_t complete)
 228 {
 229         struct ahash_request *ahreq = &pctx->u.ahreq;
 230
 231         ahash_request_set_callback(ahreq, aead_request_flags(req),
 232                                    complete, req);
 233         sg_init_one(pctx->src, gcm_zeroes, remain);
 234         ahash_request_set_crypt(ahreq, pctx->src, NULL, remain);
 235
 236         return crypto_ahash_update(ahreq);
 237 }
 238
 239 static int gcm_hash_len(struct aead_request *req,
 240                         struct crypto_gcm_req_priv_ctx *pctx)
 241 {
 242         struct ahash_request *ahreq = &pctx->u.ahreq;
 243         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 244         u128 lengths;
 245
 246         lengths.a = cpu_to_be64(req->assoclen * 8);
 247         lengths.b = cpu_to_be64(gctx->cryptlen * 8);
 248         memcpy(pctx->iauth_tag, &lengths, 16);
 249         sg_init_one(pctx->src, pctx->iauth_tag, 16);
 250         ahash_request_set_callback(ahreq, aead_request_flags(req),
 251                                    gcm_hash_len_done, req);
 252         ahash_request_set_crypt(ahreq, pctx->src,
 253                                 NULL, sizeof(lengths));
 254
 255         return crypto_ahash_update(ahreq);
 256 }
 257
 258 static int gcm_hash_final(struct aead_request *req,
 259                           struct crypto_gcm_req_priv_ctx *pctx)
 260 {
 261         struct ahash_request *ahreq = &pctx->u.ahreq;
 262
 263         ahash_request_set_callback(ahreq, aead_request_flags(req),
 264                                    gcm_hash_final_done, req);
 265         ahash_request_set_crypt(ahreq, NULL, pctx->iauth_tag, 0);
 266
 267         return crypto_ahash_final(ahreq);
 268 }
 269
 270 static void __gcm_hash_final_done(struct aead_request *req, int err)
 271 {
 272         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 273         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 274
 275         if (!err)
 276                 crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
 277
 278         gctx->complete(req, err);
 279 }
 280
 281 static void gcm_hash_final_done(struct crypto_async_request *areq, int err)
 282 {
 283         struct aead_request *req = areq->data;
 284
 285         __gcm_hash_final_done(req, err);
 286 }
 287
 288 static void __gcm_hash_len_done(struct aead_request *req, int err)
 289 {
 290         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 291
 292         if (!err) {
 293                 err = gcm_hash_final(req, pctx);
 294                 if (err == -EINPROGRESS || err == -EBUSY)
 295                         return;
 296         }
 297
 298         __gcm_hash_final_done(req, err);
 299 }
 300
 301 static void gcm_hash_len_done(struct crypto_async_request *areq, int err)
 302 {
 303         struct aead_request *req = areq->data;
 304
 305         __gcm_hash_len_done(req, err);
 306 }
 307
 308 static void __gcm_hash_crypt_remain_done(struct aead_request *req, int err)
 309 {
 310         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 311
 312         if (!err) {
 313                 err = gcm_hash_len(req, pctx);
 314                 if (err == -EINPROGRESS || err == -EBUSY)
 315                         return;
 316         }
 317
 318         __gcm_hash_len_done(req, err);
 319 }
 320
 321 static void gcm_hash_crypt_remain_done(struct crypto_async_request *areq,
 322                                        int err)
 323 {
 324         struct aead_request *req = areq->data;
 325
 326         __gcm_hash_crypt_remain_done(req, err);
 327 }
 328
 329 static void __gcm_hash_crypt_done(struct aead_request *req, int err)
 330 {
 331         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 332         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 333         unsigned int remain;
 334
 335         if (!err) {
 336                 remain = gcm_remain(gctx->cryptlen);
 337                 BUG_ON(!remain);
 338                 err = gcm_hash_remain(req, pctx, remain,
 339                                       gcm_hash_crypt_remain_done);
 340                 if (err == -EINPROGRESS || err == -EBUSY)
 341                         return;
 342         }
 343
 344         __gcm_hash_crypt_remain_done(req, err);
 345 }
 346
 347 static void gcm_hash_crypt_done(struct crypto_async_request *areq, int err)
 348 {
 349         struct aead_request *req = areq->data;
 350
 351         __gcm_hash_crypt_done(req, err);
 352 }
 353
 354 static void __gcm_hash_assoc_remain_done(struct aead_request *req, int err)
 355 {
 356         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 357         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 358         crypto_completion_t complete;
 359         unsigned int remain = 0;
 360
 361         if (!err && gctx->cryptlen) {
 362                 remain = gcm_remain(gctx->cryptlen);
 363                 complete = remain ? gcm_hash_crypt_done :
 364                         gcm_hash_crypt_remain_done;
 365                 err = gcm_hash_update(req, pctx, complete,
 366                                       gctx->src, gctx->cryptlen);
 367                 if (err == -EINPROGRESS || err == -EBUSY)
 368                         return;
 369         }
 370
 371         if (remain)
 372                 __gcm_hash_crypt_done(req, err);
 373         else
 374                 __gcm_hash_crypt_remain_done(req, err);
 375 }
 376
 377 static void gcm_hash_assoc_remain_done(struct crypto_async_request *areq,
 378                                        int err)
 379 {
 380         struct aead_request *req = areq->data;
 381
 382         __gcm_hash_assoc_remain_done(req, err);
 383 }
 384
 385 static void __gcm_hash_assoc_done(struct aead_request *req, int err)
 386 {
 387         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 388         unsigned int remain;
 389
 390         if (!err) {
 391                 remain = gcm_remain(req->assoclen);
 392                 BUG_ON(!remain);
 393                 err = gcm_hash_remain(req, pctx, remain,
 394                                       gcm_hash_assoc_remain_done);
 395                 if (err == -EINPROGRESS || err == -EBUSY)
 396                         return;
 397         }
 398
 399         __gcm_hash_assoc_remain_done(req, err);
 400 }
 401
 402 static void gcm_hash_assoc_done(struct crypto_async_request *areq, int err)
 403 {
 404         struct aead_request *req = areq->data;
 405
 406         __gcm_hash_assoc_done(req, err);
 407 }
 408
 409 static void __gcm_hash_init_done(struct aead_request *req, int err)
 410 {
 411         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 412         crypto_completion_t complete;
 413         unsigned int remain = 0;
 414
 415         if (!err && req->assoclen) {
 416                 remain = gcm_remain(req->assoclen);
 417                 complete = remain ? gcm_hash_assoc_done :
 418                         gcm_hash_assoc_remain_done;
 419                 err = gcm_hash_update(req, pctx, complete,
 420                                       req->assoc, req->assoclen);
 421                 if (err == -EINPROGRESS || err == -EBUSY)
 422                         return;
 423         }
 424
 425         if (remain)
 426                 __gcm_hash_assoc_done(req, err);
 427         else
 428                 __gcm_hash_assoc_remain_done(req, err);
 429 }
 430
 431 static void gcm_hash_init_done(struct crypto_async_request *areq, int err)
 432 {
 433         struct aead_request *req = areq->data;
 434
 435         __gcm_hash_init_done(req, err);
 436 }
 437
 438 static int gcm_hash(struct aead_request *req,
 439                     struct crypto_gcm_req_priv_ctx *pctx)
 440 {
 441         struct ahash_request *ahreq = &pctx->u.ahreq;
 442         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 443         struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(req->base.tfm);
 444         unsigned int remain;
 445         crypto_completion_t complete;
 446         int err;
 447
 448         ahash_request_set_tfm(ahreq, ctx->ghash);
 449
 450         ahash_request_set_callback(ahreq, aead_request_flags(req),
 451                                    gcm_hash_init_done, req);
 452         err = crypto_ahash_init(ahreq);
 453         if (err)
 454                 return err;
 455         remain = gcm_remain(req->assoclen);
 456         complete = remain ? gcm_hash_assoc_done : gcm_hash_assoc_remain_done;
 457         err = gcm_hash_update(req, pctx, complete, req->assoc, req->assoclen);
 458         if (err)
 459                 return err;
 460         if (remain) {
 461                 err = gcm_hash_remain(req, pctx, remain,
 462                                       gcm_hash_assoc_remain_done);
 463                 if (err)
 464                         return err;
 465         }
 466         remain = gcm_remain(gctx->cryptlen);
 467         complete = remain ? gcm_hash_crypt_done : gcm_hash_crypt_remain_done;
 468         err = gcm_hash_update(req, pctx, complete, gctx->src, gctx->cryptlen);
 469         if (err)
 470                 return err;
 471         if (remain) {
 472                 err = gcm_hash_remain(req, pctx, remain,
 473                                       gcm_hash_crypt_remain_done);
 474                 if (err)
 475                         return err;
 476         }
 477         err = gcm_hash_len(req, pctx);
 478         if (err)
 479                 return err;
 480         err = gcm_hash_final(req, pctx);
 481         if (err)
 482                 return err;
 483
 484         return 0;
 485 }
 486
 487 static void gcm_enc_copy_hash(struct aead_request *req,
 488                               struct crypto_gcm_req_priv_ctx *pctx)
 489 {
 490         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 491         u8 *auth_tag = pctx->auth_tag;
 492
 493         scatterwalk_map_and_copy(auth_tag, req->dst, req->cryptlen,
 494                                  crypto_aead_authsize(aead), 1);
 495 }
 496
 497 static void gcm_enc_hash_done(struct aead_request *req, int err)
 498 {
 499         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 500
 501         if (!err)
 502                 gcm_enc_copy_hash(req, pctx);
 503
 504         aead_request_complete(req, err);
 505 }
 506
 507 static void gcm_encrypt_done(struct crypto_async_request *areq, int err)
 508 {
 509         struct aead_request *req = areq->data;
 510         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 511
 512         if (!err) {
 513                 err = gcm_hash(req, pctx);
 514                 if (err == -EINPROGRESS || err == -EBUSY)
 515                         return;
 516                 else if (!err) {
 517                         crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
 518                         gcm_enc_copy_hash(req, pctx);
 519                 }
 520         }
 521
 522         aead_request_complete(req, err);
 523 }
 524
 525 static int crypto_gcm_encrypt(struct aead_request *req)
 526 {
 527         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 528         struct ablkcipher_request *abreq = &pctx->u.abreq;
 529         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 530         int err;
 531
 532         crypto_gcm_init_crypt(abreq, req, req->cryptlen);
 533         ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 534                                         gcm_encrypt_done, req);
 535
 536         gctx->src = req->dst;
 537         gctx->cryptlen = req->cryptlen;
 538         gctx->complete = gcm_enc_hash_done;
 539
 540         err = crypto_ablkcipher_encrypt(abreq);
 541         if (err)
 542                 return err;
 543
 544         err = gcm_hash(req, pctx);
 545         if (err)
 546                 return err;
 547
 548         crypto_xor(pctx->auth_tag, pctx->iauth_tag, 16);
 549         gcm_enc_copy_hash(req, pctx);
 550
 551         return 0;
 552 }
 553
 554 static int crypto_gcm_verify(struct aead_request *req,
 555                              struct crypto_gcm_req_priv_ctx *pctx)
 556 {
 557         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 558         u8 *auth_tag = pctx->auth_tag;
 559         u8 *iauth_tag = pctx->iauth_tag;
 560         unsigned int authsize = crypto_aead_authsize(aead);
 561         unsigned int cryptlen = req->cryptlen - authsize;
 562
 563         crypto_xor(auth_tag, iauth_tag, 16);
 564         scatterwalk_map_and_copy(iauth_tag, req->src, cryptlen, authsize, 0);
 565         return memcmp(iauth_tag, auth_tag, authsize) ? -EBADMSG : 0;
 566 }
 567
 568 static void gcm_decrypt_done(struct crypto_async_request *areq, int err)
 569 {
 570         struct aead_request *req = areq->data;
 571         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 572
 573         if (!err)
 574                 err = crypto_gcm_verify(req, pctx);
 575
 576         aead_request_complete(req, err);
 577 }
 578
 579 static void gcm_dec_hash_done(struct aead_request *req, int err)
 580 {
 581         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 582         struct ablkcipher_request *abreq = &pctx->u.abreq;
 583         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 584
 585         if (!err) {
 586                 ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 587                                                 gcm_decrypt_done, req);
 588                 crypto_gcm_init_crypt(abreq, req, gctx->cryptlen);
 589                 err = crypto_ablkcipher_decrypt(abreq);
 590                 if (err == -EINPROGRESS || err == -EBUSY)
 591                         return;
 592                 else if (!err)
 593                         err = crypto_gcm_verify(req, pctx);
 594         }
 595
 596         aead_request_complete(req, err);
 597 }
 598
 599 static int crypto_gcm_decrypt(struct aead_request *req)
 600 {
 601         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 602         struct crypto_gcm_req_priv_ctx *pctx = crypto_gcm_reqctx(req);
 603         struct ablkcipher_request *abreq = &pctx->u.abreq;
 604         struct crypto_gcm_ghash_ctx *gctx = &pctx->ghash_ctx;
 605         unsigned int authsize = crypto_aead_authsize(aead);
 606         unsigned int cryptlen = req->cryptlen;
 607         int err;
 608
 609         if (cryptlen < authsize)
 610                 return -EINVAL;
 611         cryptlen -= authsize;
 612
 613         gctx->src = req->src;
 614         gctx->cryptlen = cryptlen;
 615         gctx->complete = gcm_dec_hash_done;
 616
 617         err = gcm_hash(req, pctx);
 618         if (err)
 619                 return err;
 620
 621         ablkcipher_request_set_callback(abreq, aead_request_flags(req),
 622                                         gcm_decrypt_done, req);
 623         crypto_gcm_init_crypt(abreq, req, cryptlen);
 624         err = crypto_ablkcipher_decrypt(abreq);
 625         if (err)
 626                 return err;
 627
 628         return crypto_gcm_verify(req, pctx);
 629 }
 630
 631 static int crypto_gcm_init_tfm(struct crypto_tfm *tfm)
 632 {
 633         struct crypto_instance *inst = (void *)tfm->__crt_alg;
 634         struct gcm_instance_ctx *ictx = crypto_instance_ctx(inst);
 635         struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 636         struct crypto_ablkcipher *ctr;
 637         struct crypto_ahash *ghash;
 638         unsigned long align;
 639         int err;
 640
 641         ghash = crypto_spawn_ahash(&ictx->ghash);
 642         if (IS_ERR(ghash))
 643                 return PTR_ERR(ghash);
 644
 645         ctr = crypto_spawn_skcipher(&ictx->ctr);
 646         err = PTR_ERR(ctr);
 647         if (IS_ERR(ctr))
 648                 goto err_free_hash;
 649
 650         ctx->ctr = ctr;
 651         ctx->ghash = ghash;
 652
 653         align = crypto_tfm_alg_alignmask(tfm);
 654         align &= ~(crypto_tfm_ctx_alignment() - 1);
 655         tfm->crt_aead.reqsize = align +
 656                 offsetof(struct crypto_gcm_req_priv_ctx, u) +
 657                 max(sizeof(struct ablkcipher_request) +
 658                     crypto_ablkcipher_reqsize(ctr),
 659                     sizeof(struct ahash_request) +
 660                     crypto_ahash_reqsize(ghash));
 661
 662         return 0;
 663
 664 err_free_hash:
 665         crypto_free_ahash(ghash);
 666         return err;
 667 }
 668
 669 static void crypto_gcm_exit_tfm(struct crypto_tfm *tfm)
 670 {
 671         struct crypto_gcm_ctx *ctx = crypto_tfm_ctx(tfm);
 672
 673         crypto_free_ahash(ctx->ghash);
 674         crypto_free_ablkcipher(ctx->ctr);
 675 }
 676
 677 static struct crypto_instance *crypto_gcm_alloc_common(struct rtattr **tb,
 678                                                        const char *full_name,
 679                                                        const char *ctr_name,
 680                                                        const char *ghash_name)
 681 {
 682         struct crypto_attr_type *algt;
 683         struct crypto_instance *inst;
 684         struct crypto_alg *ctr;
 685         struct crypto_alg *ghash_alg;
 686         struct ahash_alg *ghash_ahash_alg;
 687         struct gcm_instance_ctx *ctx;
 688         int err;
 689
 690         algt = crypto_get_attr_type(tb);
 691         err = PTR_ERR(algt);
 692         if (IS_ERR(algt))
 693                 return ERR_PTR(err);
 694
 695         if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 696                 return ERR_PTR(-EINVAL);
 697
 698         ghash_alg = crypto_find_alg(ghash_name, &crypto_ahash_type,
 699                                     CRYPTO_ALG_TYPE_HASH,
 700                                     CRYPTO_ALG_TYPE_AHASH_MASK);
 701         err = PTR_ERR(ghash_alg);
 702         if (IS_ERR(ghash_alg))
 703                 return ERR_PTR(err);
 704
 705         err = -ENOMEM;
 706         inst = kzalloc(sizeof(*inst) + sizeof(*ctx), GFP_KERNEL);
 707         if (!inst)
 708                 goto out_put_ghash;
 709
 710         ctx = crypto_instance_ctx(inst);
 711         ghash_ahash_alg = container_of(ghash_alg, struct ahash_alg, halg.base);
 712         err = crypto_init_ahash_spawn(&ctx->ghash, &ghash_ahash_alg->halg,
 713                                       inst);
 714         if (err)
 715                 goto err_free_inst;
 716
 717         crypto_set_skcipher_spawn(&ctx->ctr, inst);
 718         err = crypto_grab_skcipher(&ctx->ctr, ctr_name, 0,
 719                                    crypto_requires_sync(algt->type,
 720                                                         algt->mask));
 721         if (err)
 722                 goto err_drop_ghash;
 723
 724         ctr = crypto_skcipher_spawn_alg(&ctx->ctr);
 725
 726         /* We only support 16-byte blocks. */
 727         if (ctr->cra_ablkcipher.ivsize != 16)
 728                 goto out_put_ctr;
 729
 730         /* Not a stream cipher? */
 731         err = -EINVAL;
 732         if (ctr->cra_blocksize != 1)
 733                 goto out_put_ctr;
 734
 735         err = -ENAMETOOLONG;
 736         if (snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
 737                      "gcm_base(%s,%s)", ctr->cra_driver_name,
 738                      ghash_alg->cra_driver_name) >=
 739             CRYPTO_MAX_ALG_NAME)
 740                 goto out_put_ctr;
 741
 742         memcpy(inst->alg.cra_name, full_name, CRYPTO_MAX_ALG_NAME);
 743
 744         inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
 745         inst->alg.cra_flags |= ctr->cra_flags & CRYPTO_ALG_ASYNC;
 746         inst->alg.cra_priority = ctr->cra_priority;
 747         inst->alg.cra_blocksize = 1;
 748         inst->alg.cra_alignmask = ctr->cra_alignmask | (__alignof__(u64) - 1);
 749         inst->alg.cra_type = &crypto_aead_type;
 750         inst->alg.cra_aead.ivsize = 16;
 751         inst->alg.cra_aead.maxauthsize = 16;
 752         inst->alg.cra_ctxsize = sizeof(struct crypto_gcm_ctx);
 753         inst->alg.cra_init = crypto_gcm_init_tfm;
 754         inst->alg.cra_exit = crypto_gcm_exit_tfm;
 755         inst->alg.cra_aead.setkey = crypto_gcm_setkey;
 756         inst->alg.cra_aead.setauthsize = crypto_gcm_setauthsize;
 757         inst->alg.cra_aead.encrypt = crypto_gcm_encrypt;
 758         inst->alg.cra_aead.decrypt = crypto_gcm_decrypt;
 759
 760 out:
 761         crypto_mod_put(ghash_alg);
 762         return inst;
 763
 764 out_put_ctr:
 765         crypto_drop_skcipher(&ctx->ctr);
 766 err_drop_ghash:
 767         crypto_drop_ahash(&ctx->ghash);
 768 err_free_inst:
 769         kfree(inst);
 770 out_put_ghash:
 771         inst = ERR_PTR(err);
 772         goto out;
 773 }
 774
 775 static struct crypto_instance *crypto_gcm_alloc(struct rtattr **tb)
 776 {
 777         int err;
 778         const char *cipher_name;
 779         char ctr_name[CRYPTO_MAX_ALG_NAME];
 780         char full_name[CRYPTO_MAX_ALG_NAME];
 781
 782         cipher_name = crypto_attr_alg_name(tb[1]);
 783         err = PTR_ERR(cipher_name);
 784         if (IS_ERR(cipher_name))
 785                 return ERR_PTR(err);
 786
 787         if (snprintf(ctr_name, CRYPTO_MAX_ALG_NAME, "ctr(%s)", cipher_name) >=
 788             CRYPTO_MAX_ALG_NAME)
 789                 return ERR_PTR(-ENAMETOOLONG);
 790
 791         if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm(%s)", cipher_name) >=
 792             CRYPTO_MAX_ALG_NAME)
 793                 return ERR_PTR(-ENAMETOOLONG);
 794
 795         return crypto_gcm_alloc_common(tb, full_name, ctr_name, "ghash");
 796 }
 797
 798 static void crypto_gcm_free(struct crypto_instance *inst)
 799 {
 800         struct gcm_instance_ctx *ctx = crypto_instance_ctx(inst);
 801
 802         crypto_drop_skcipher(&ctx->ctr);
 803         crypto_drop_ahash(&ctx->ghash);
 804         kfree(inst);
 805 }
 806
 807 static struct crypto_template crypto_gcm_tmpl = {
 808         .name = "gcm",
 809         .alloc = crypto_gcm_alloc,
 810         .free = crypto_gcm_free,
 811         .module = THIS_MODULE,
 812 };
 813
 814 static struct crypto_instance *crypto_gcm_base_alloc(struct rtattr **tb)
 815 {
 816         int err;
 817         const char *ctr_name;
 818         const char *ghash_name;
 819         char full_name[CRYPTO_MAX_ALG_NAME];
 820
 821         ctr_name = crypto_attr_alg_name(tb[1]);
 822         err = PTR_ERR(ctr_name);
 823         if (IS_ERR(ctr_name))
 824                 return ERR_PTR(err);
 825
 826         ghash_name = crypto_attr_alg_name(tb[2]);
 827         err = PTR_ERR(ghash_name);
 828         if (IS_ERR(ghash_name))
 829                 return ERR_PTR(err);
 830
 831         if (snprintf(full_name, CRYPTO_MAX_ALG_NAME, "gcm_base(%s,%s)",
 832                      ctr_name, ghash_name) >= CRYPTO_MAX_ALG_NAME)
 833                 return ERR_PTR(-ENAMETOOLONG);
 834
 835         return crypto_gcm_alloc_common(tb, full_name, ctr_name, ghash_name);
 836 }
 837
 838 static struct crypto_template crypto_gcm_base_tmpl = {
 839         .name = "gcm_base",
 840         .alloc = crypto_gcm_base_alloc,
 841         .free = crypto_gcm_free,
 842         .module = THIS_MODULE,
 843 };
 844
 845 static int crypto_rfc4106_setkey(struct crypto_aead *parent, const u8 *key,
 846                                  unsigned int keylen)
 847 {
 848         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
 849         struct crypto_aead *child = ctx->child;
 850         int err;
 851
 852         if (keylen < 4)
 853                 return -EINVAL;
 854
 855         keylen -= 4;
 856         memcpy(ctx->nonce, key + keylen, 4);
 857
 858         crypto_aead_clear_flags(child, CRYPTO_TFM_REQ_MASK);
 859         crypto_aead_set_flags(child, crypto_aead_get_flags(parent) &
 860                                      CRYPTO_TFM_REQ_MASK);
 861         err = crypto_aead_setkey(child, key, keylen);
 862         crypto_aead_set_flags(parent, crypto_aead_get_flags(child) &
 863                                       CRYPTO_TFM_RES_MASK);
 864
 865         return err;
 866 }
 867
 868 static int crypto_rfc4106_setauthsize(struct crypto_aead *parent,
 869                                       unsigned int authsize)
 870 {
 871         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(parent);
 872
 873         switch (authsize) {
 874         case 8:
 875         case 12:
 876         case 16:
 877                 break;
 878         default:
 879                 return -EINVAL;
 880         }
 881
 882         return crypto_aead_setauthsize(ctx->child, authsize);
 883 }
 884
 885 static struct aead_request *crypto_rfc4106_crypt(struct aead_request *req)
 886 {
 887         struct aead_request *subreq = aead_request_ctx(req);
 888         struct crypto_aead *aead = crypto_aead_reqtfm(req);
 889         struct crypto_rfc4106_ctx *ctx = crypto_aead_ctx(aead);
 890         struct crypto_aead *child = ctx->child;
 891         u8 *iv = PTR_ALIGN((u8 *)(subreq + 1) + crypto_aead_reqsize(child),
 892                            crypto_aead_alignmask(child) + 1);
 893
 894         memcpy(iv, ctx->nonce, 4);
 895         memcpy(iv + 4, req->iv, 8);
 896
 897         aead_request_set_tfm(subreq, child);
 898         aead_request_set_callback(subreq, req->base.flags, req->base.complete,
 899                                   req->base.data);
 900         aead_request_set_crypt(subreq, req->src, req->dst, req->cryptlen, iv);
 901         aead_request_set_assoc(subreq, req->assoc, req->assoclen);
 902
 903         return subreq;
 904 }
 905
 906 static int crypto_rfc4106_encrypt(struct aead_request *req)
 907 {
 908         req = crypto_rfc4106_crypt(req);
 909
 910         return crypto_aead_encrypt(req);
 911 }
 912
 913 static int crypto_rfc4106_decrypt(struct aead_request *req)
 914 {
 915         req = crypto_rfc4106_crypt(req);
 916
 917         return crypto_aead_decrypt(req);
 918 }
 919
 920 static int crypto_rfc4106_init_tfm(struct crypto_tfm *tfm)
 921 {
 922         struct crypto_instance *inst = (void *)tfm->__crt_alg;
 923         struct crypto_aead_spawn *spawn = crypto_instance_ctx(inst);
 924         struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm);
 925         struct crypto_aead *aead;
 926         unsigned long align;
 927
 928         aead = crypto_spawn_aead(spawn);
 929         if (IS_ERR(aead))
 930                 return PTR_ERR(aead);
 931
 932         ctx->child = aead;
 933
 934         align = crypto_aead_alignmask(aead);
 935         align &= ~(crypto_tfm_ctx_alignment() - 1);
 936         tfm->crt_aead.reqsize = sizeof(struct aead_request) +
 937                                 ALIGN(crypto_aead_reqsize(aead),
 938                                       crypto_tfm_ctx_alignment()) +
 939                                 align + 16;
 940
 941         return 0;
 942 }
 943
 944 static void crypto_rfc4106_exit_tfm(struct crypto_tfm *tfm)
 945 {
 946         struct crypto_rfc4106_ctx *ctx = crypto_tfm_ctx(tfm);
 947
 948         crypto_free_aead(ctx->child);
 949 }
 950
 951 static struct crypto_instance *crypto_rfc4106_alloc(struct rtattr **tb)
 952 {
 953         struct crypto_attr_type *algt;
 954         struct crypto_instance *inst;
 955         struct crypto_aead_spawn *spawn;
 956         struct crypto_alg *alg;
 957         const char *ccm_name;
 958         int err;
 959
 960         algt = crypto_get_attr_type(tb);
 961         err = PTR_ERR(algt);
 962         if (IS_ERR(algt))
 963                 return ERR_PTR(err);
 964
 965         if ((algt->type ^ CRYPTO_ALG_TYPE_AEAD) & algt->mask)
 966                 return ERR_PTR(-EINVAL);
 967
 968         ccm_name = crypto_attr_alg_name(tb[1]);
 969         err = PTR_ERR(ccm_name);
 970         if (IS_ERR(ccm_name))
 971                 return ERR_PTR(err);
 972
 973         inst = kzalloc(sizeof(*inst) + sizeof(*spawn), GFP_KERNEL);
 974         if (!inst)
 975                 return ERR_PTR(-ENOMEM);
 976
 977         spawn = crypto_instance_ctx(inst);
 978         crypto_set_aead_spawn(spawn, inst);
 979         err = crypto_grab_aead(spawn, ccm_name, 0,
 980                                crypto_requires_sync(algt->type, algt->mask));
 981         if (err)
 982                 goto out_free_inst;
 983
 984         alg = crypto_aead_spawn_alg(spawn);
 985
 986         err = -EINVAL;
 987
 988         /* We only support 16-byte blocks. */
 989         if (alg->cra_aead.ivsize != 16)
 990                 goto out_drop_alg;
 991
 992         /* Not a stream cipher? */
 993         if (alg->cra_blocksize != 1)
 994                 goto out_drop_alg;
 995
 996         err = -ENAMETOOLONG;
 997         if (snprintf(inst->alg.cra_name, CRYPTO_MAX_ALG_NAME,
 998                      "rfc4106(%s)", alg->cra_name) >= CRYPTO_MAX_ALG_NAME ||
 999             snprintf(inst->alg.cra_driver_name, CRYPTO_MAX_ALG_NAME,
1000                      "rfc4106(%s)", alg->cra_driver_name) >=
1001             CRYPTO_MAX_ALG_NAME)
1002                 goto out_drop_alg;
1003
1004         inst->alg.cra_flags = CRYPTO_ALG_TYPE_AEAD;
1005         inst->alg.cra_flags |= alg->cra_flags & CRYPTO_ALG_ASYNC;
1006         inst->alg.cra_priority = alg->cra_priority;
1007         inst->alg.cra_blocksize = 1;
1008         inst->alg.cra_alignmask = alg->cra_alignmask;
1009         inst->alg.cra_type = &crypto_nivaead_type;
1010
1011         inst->alg.cra_aead.ivsize = 8;
1012         inst->alg.cra_aead.maxauthsize = 16;
1013
1014         inst->alg.cra_ctxsize = sizeof(struct crypto_rfc4106_ctx);
1015
1016         inst->alg.cra_init = crypto_rfc4106_init_tfm;
1017         inst->alg.cra_exit = crypto_rfc4106_exit_tfm;
1018
1019         inst->alg.cra_aead.setkey = crypto_rfc4106_setkey;
1020         inst->alg.cra_aead.setauthsize = crypto_rfc4106_setauthsize;
1021         inst->alg.cra_aead.encrypt = crypto_rfc4106_encrypt;
1022         inst->alg.cra_aead.decrypt = crypto_rfc4106_decrypt;
1023
1024         inst->alg.cra_aead.geniv = "seqiv";
1025
1026 out:
1027         return inst;
1028
1029 out_drop_alg:
1030         crypto_drop_aead(spawn);
1031 out_free_inst:
1032         kfree(inst);
1033         inst = ERR_PTR(err);
1034         goto out;
1035 }
1036
1037 static void crypto_rfc4106_free(struct crypto_instance *inst)
1038 {
1039         crypto_drop_spawn(crypto_instance_ctx(inst));
1040         kfree(inst);
1041 }
1042
1043 static struct crypto_template crypto_rfc4106_tmpl = {
1044         .name = "rfc4106",
1045         .alloc = crypto_rfc4106_alloc,
1046         .free = crypto_rfc4106_free,
1047         .module = THIS_MODULE,
1048 };
1049
1050 static int __init crypto_gcm_module_init(void)
1051 {
1052         int err;
1053
1054         gcm_zeroes = kzalloc(16, GFP_KERNEL);
1055         if (!gcm_zeroes)
1056                 return -ENOMEM;
1057
1058         err = crypto_register_template(&crypto_gcm_base_tmpl);
1059         if (err)
1060                 goto out;
1061
1062         err = crypto_register_template(&crypto_gcm_tmpl);
1063         if (err)
1064                 goto out_undo_base;
1065
1066         err = crypto_register_template(&crypto_rfc4106_tmpl);
1067         if (err)
1068                 goto out_undo_gcm;
1069
1070         return 0;
1071
1072 out_undo_gcm:
1073         crypto_unregister_template(&crypto_gcm_tmpl);
1074 out_undo_base:
1075         crypto_unregister_template(&crypto_gcm_base_tmpl);
1076 out:
1077         kfree(gcm_zeroes);
1078         return err;
1079 }
1080
1081 static void __exit crypto_gcm_module_exit(void)
1082 {
1083         kfree(gcm_zeroes);
1084         crypto_unregister_template(&crypto_rfc4106_tmpl);
1085         crypto_unregister_template(&crypto_gcm_tmpl);
1086         crypto_unregister_template(&crypto_gcm_base_tmpl);
1087 }
1088
1089 module_init(crypto_gcm_module_init);
1090 module_exit(crypto_gcm_module_exit);
1091
1092 MODULE_LICENSE("GPL");
1093 MODULE_DESCRIPTION("Galois/Counter Mode");
1094 MODULE_AUTHOR("Mikko Herranen <mh1@iki.fi>");
1095 MODULE_ALIAS("gcm_base");
1096 MODULE_ALIAS("rfc4106");