main.c

   1 /*
   2  * Driver for /dev/crypto device (aka CryptoDev)
   3  *
   4  * Copyright (c) 2004 Michal Ludvig <mludvig@logix.net.nz>, SuSE Labs
   5  * Copyright (c) 2009,2010 Nikos Mavrogiannopoulos <nmav@gnutls.org>
   6  *
   7  * This file is part of linux cryptodev.
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * as published by the Free Software Foundation; either version 2
  12  * of the License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc.,
  22  * 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
  23  */
  24
  25 /*
  26  * Device /dev/crypto provides an interface for
  27  * accessing kernel CryptoAPI algorithms (ciphers,
  28  * hashes) from userspace programs.
  29  *
  30  * /dev/crypto interface was originally introduced in
  31  * OpenBSD and this module attempts to keep the API.
  32  *
  33  */
  34 #include <crypto/hash.h>
  35 #include <linux/crypto.h>
  36 #include <linux/mm.h>
  37 #include <linux/highmem.h>
  38 #include <linux/ioctl.h>
  39 #include <linux/random.h>
  40 #include <linux/syscalls.h>
  41 #include <linux/pagemap.h>
  42 #include <linux/poll.h>
  43 #include <linux/uaccess.h>
  44 #include <crypto/cryptodev.h>
  45 #include <crypto/scatterwalk.h>
  46 #include <linux/scatterlist.h>
  47 #include "cryptodev_int.h"
  48 #include "zc.h"
  49 #include "version.h"
  50
  51 /* This file contains the traditional operations of encryption
  52  * and hashing of /dev/crypto.
  53  */
  54
  55 static int
  56 hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop,
  57                 struct scatterlist *src_sg, struct scatterlist *dst_sg,
  58                 uint32_t len)
  59 {
  60         int ret;
  61
  62         /* Always hash before encryption and after decryption. Maybe
  63          * we should introduce a flag to switch... TBD later on.
  64          */
  65         if (cop->op == COP_ENCRYPT) {
  66                 if (ses_ptr->hdata.init != 0) {
  67                         ret = cryptodev_hash_update(&ses_ptr->hdata,
  68                                                         src_sg, len);
  69                         if (unlikely(ret))
  70                                 goto out_err;
  71                 }
  72                 if (ses_ptr->cdata.init != 0) {
  73                         ret = cryptodev_cipher_encrypt(&ses_ptr->cdata,
  74                                                         src_sg, dst_sg, len);
  75
  76                         if (unlikely(ret))
  77                                 goto out_err;
  78                 }
  79         } else {
  80                 if (ses_ptr->cdata.init != 0) {
  81                         ret = cryptodev_cipher_decrypt(&ses_ptr->cdata,
  82                                                         src_sg, dst_sg, len);
  83
  84                         if (unlikely(ret))
  85                                 goto out_err;
  86                 }
  87
  88                 if (ses_ptr->hdata.init != 0) {
  89                         ret = cryptodev_hash_update(&ses_ptr->hdata,
  90                                                                 dst_sg, len);
  91                         if (unlikely(ret))
  92                                 goto out_err;
  93                 }
  94         }
  95         return 0;
  96 out_err:
  97         dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
  98         return ret;
  99 }
 100
 101 /* This is the main crypto function - feed it with plaintext
 102    and get a ciphertext (or vice versa :-) */
 103 static int
 104 __crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop)
 105 {
 106         char *data;
 107         char __user *src, *dst;
 108         struct scatterlist sg;
 109         size_t nbytes, bufsize;
 110         int ret = 0;
 111
 112         nbytes = cop->len;
 113         data = (char *)__get_free_page(GFP_KERNEL);
 114
 115         if (unlikely(!data))
 116                 return -ENOMEM;
 117
 118         bufsize = PAGE_SIZE < nbytes ? PAGE_SIZE : nbytes;
 119
 120         src = cop->src;
 121         dst = cop->dst;
 122
 123         while (nbytes > 0) {
 124                 size_t current_len = nbytes > bufsize ? bufsize : nbytes;
 125
 126                 if (unlikely(copy_from_user(data, src, current_len))) {
 127                         ret = -EFAULT;
 128                         break;
 129                 }
 130
 131                 sg_init_one(&sg, data, current_len);
 132
 133                 ret = hash_n_crypt(ses_ptr, cop, &sg, &sg, current_len);
 134
 135                 if (unlikely(ret))
 136                         break;
 137
 138                 if (ses_ptr->cdata.init != 0) {
 139                         if (unlikely(copy_to_user(dst, data, current_len))) {
 140                                 ret = -EFAULT;
 141                                 break;
 142                         }
 143                 }
 144
 145                 dst += current_len;
 146                 nbytes -= current_len;
 147                 src += current_len;
 148         }
 149
 150         free_page((unsigned long)data);
 151         return ret;
 152 }
 153
 154 /* make cop->src and cop->dst available in scatterlists */
 155 static int get_userbuf(struct csession *ses, struct kernel_crypt_op *kcop,
 156                        struct scatterlist **src_sg, struct scatterlist **dst_sg,
 157                        int *tot_pages)
 158 {
 159         int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1;
 160         struct crypt_op *cop = &kcop->cop;
 161         int rc;
 162
 163         if (cop->src == NULL)
 164                 return -EINVAL;
 165
 166         if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->src, ses->alignmask)) {
 167                 dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
 168                                 __func__, (unsigned long)cop->src, ses->alignmask + 1);
 169         }
 170
 171         src_pagecount = PAGECOUNT(cop->src, cop->len);
 172         if (!ses->cdata.init) {         /* hashing only */
 173                 write_src = 0;
 174         } else if (cop->src != cop->dst) {      /* non-in-situ transformation */
 175                 if (cop->dst == NULL)
 176                         return -EINVAL;
 177
 178                 dst_pagecount = PAGECOUNT(cop->dst, cop->len);
 179                 write_src = 0;
 180
 181                 if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->dst, ses->alignmask)) {
 182                         dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n",
 183                                         __func__, (unsigned long)cop->dst, ses->alignmask + 1);
 184                 }
 185
 186         }
 187         (*tot_pages) = pagecount = src_pagecount + dst_pagecount;
 188
 189         if (pagecount > ses->array_size) {
 190                 rc = adjust_sg_array(ses, pagecount);
 191                 if (rc)
 192                         return rc;
 193         }
 194
 195         rc = __get_userbuf(cop->src, cop->len, write_src, src_pagecount,
 196                            ses->pages, ses->sg, kcop->task, kcop->mm);
 197         if (unlikely(rc)) {
 198                 dprintk(1, KERN_ERR,
 199                         "failed to get user pages for data input\n");
 200                 return -EINVAL;
 201         }
 202         (*src_sg) = (*dst_sg) = ses->sg;
 203
 204         if (!dst_pagecount)
 205                 return 0;
 206
 207         (*dst_sg) = ses->sg + src_pagecount;
 208
 209         rc = __get_userbuf(cop->dst, cop->len, 1, dst_pagecount,
 210                            ses->pages + src_pagecount, *dst_sg,
 211                            kcop->task, kcop->mm);
 212         if (unlikely(rc)) {
 213                 dprintk(1, KERN_ERR,
 214                         "failed to get user pages for data output\n");
 215                 release_user_pages(ses->pages, src_pagecount);
 216                 return -EINVAL;
 217         }
 218         return 0;
 219 }
 220
 221
 222 /* This is the main crypto function - zero-copy edition */
 223 static int
 224 __crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop)
 225 {
 226         struct scatterlist *src_sg, *dst_sg;
 227         struct crypt_op *cop = &kcop->cop;
 228         int ret = 0, pagecount;
 229
 230         ret = get_userbuf(ses_ptr, kcop, &src_sg, &dst_sg, &pagecount);
 231         if (unlikely(ret)) {
 232                 dprintk(1, KERN_ERR, "Error getting user pages. "
 233                                         "Falling back to non zero copy.\n");
 234                 return __crypto_run_std(ses_ptr, cop);
 235         }
 236
 237         ret = hash_n_crypt(ses_ptr, cop, src_sg, dst_sg, cop->len);
 238
 239         release_user_pages(ses_ptr->pages, pagecount);
 240         return ret;
 241 }
 242
 243 int crypto_run(struct fcrypt *fcr, struct kernel_crypt_op *kcop)
 244 {
 245         struct csession *ses_ptr;
 246         struct crypt_op *cop = &kcop->cop;
 247         int ret;
 248
 249         if (unlikely(cop->op != COP_ENCRYPT && cop->op != COP_DECRYPT)) {
 250                 dprintk(1, KERN_DEBUG, "invalid operation op=%u\n", cop->op);
 251                 return -EINVAL;
 252         }
 253
 254         /* this also enters ses_ptr->sem */
 255         ses_ptr = crypto_get_session_by_sid(fcr, cop->ses);
 256         if (unlikely(!ses_ptr)) {
 257                 dprintk(1, KERN_ERR, "invalid session ID=0x%08X\n", cop->ses);
 258                 return -EINVAL;
 259         }
 260
 261         if (ses_ptr->hdata.init != 0 && !(cop->flags & (COP_FLAG_UPDATE | COP_FLAG_FINAL))) {
 262                 ret = cryptodev_hash_reset(&ses_ptr->hdata);
 263                 if (unlikely(ret)) {
 264                         dprintk(1, KERN_ERR,
 265                                 "error in cryptodev_hash_reset()\n");
 266                         goto out_unlock;
 267                 }
 268         }
 269
 270         if (ses_ptr->cdata.init != 0) {
 271                 int blocksize = ses_ptr->cdata.blocksize;
 272
 273                 if (unlikely(cop->len % blocksize)) {
 274                         dprintk(1, KERN_ERR,
 275                                 "data size (%u) isn't a multiple "
 276                                 "of block size (%u)\n",
 277                                 cop->len, blocksize);
 278                         ret = -EINVAL;
 279                         goto out_unlock;
 280                 }
 281
 282                 cryptodev_cipher_set_iv(&ses_ptr->cdata, kcop->iv,
 283                                 min(ses_ptr->cdata.ivsize, kcop->ivlen));
 284         }
 285
 286         if (likely(cop->len)) {
 287                 if (cop->flags & COP_FLAG_NO_ZC)
 288                         ret = __crypto_run_std(ses_ptr, &kcop->cop);
 289                 else
 290                         ret = __crypto_run_zc(ses_ptr, kcop);
 291                 if (unlikely(ret))
 292                         goto out_unlock;
 293         }
 294
 295         if (ses_ptr->cdata.init != 0) {
 296                 cryptodev_cipher_get_iv(&ses_ptr->cdata, kcop->iv,
 297                                 min(ses_ptr->cdata.ivsize, kcop->ivlen));
 298         }
 299
 300         if (ses_ptr->hdata.init != 0 &&
 301                 ((cop->flags & COP_FLAG_FINAL) ||
 302                    (!(cop->flags & COP_FLAG_UPDATE) || cop->len == 0))) {
 303
 304                 ret = cryptodev_hash_final(&ses_ptr->hdata, kcop->hash_output);
 305                 if (unlikely(ret)) {
 306                         dprintk(0, KERN_ERR, "CryptoAPI failure: %d\n", ret);
 307                         goto out_unlock;
 308                 }
 309                 kcop->digestsize = ses_ptr->hdata.digestsize;
 310         }
 311
 312 out_unlock:
 313         crypto_put_session(ses_ptr);
 314         return ret;
 315 }