zc.c

   1 /*
   2  * Driver for /dev/crypto device (aka CryptoDev)
   3  *
   4  * Copyright (c) 2009-2011 Nikos Mavrogiannopoulos <nmav@gnutls.org>
   5  * Copyright (c) 2010 Phil Sutter
   6  *
   7  * This file is part of linux cryptodev.
   8  *
   9  * This program is free software; you can redistribute it and/or
  10  * modify it under the terms of the GNU General Public License
  11  * as published by the Free Software Foundation; either version 2
  12  * of the License, or (at your option) any later version.
  13  *
  14  * This program is distributed in the hope that it will be useful,
  15  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  16  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  17  * GNU General Public License for more details.
  18  *
  19  * You should have received a copy of the GNU General Public License
  20  * along with this program; if not, write to the Free Software
  21  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
  22  * 02110-1301, USA.
  23  */
  24
  25 #include <crypto/hash.h>
  26 #include <linux/crypto.h>
  27 #include <linux/mm.h>
  28 #include <linux/highmem.h>
  29 #include <linux/ioctl.h>
  30 #include <linux/random.h>
  31 #include <linux/syscalls.h>
  32 #include <linux/pagemap.h>
  33 #include <linux/uaccess.h>
  34 #include <crypto/scatterwalk.h>
  35 #include <linux/scatterlist.h>
  36 #include "cryptodev_int.h"
  37 #include "zc.h"
  38 #include "version.h"
  39
  40 /* Helper functions to assist zero copy.
  41  * This needs to be redesigned and moved out of the session. --nmav
  42  */
  43
  44 /* offset of buf in it's first page */
  45 #define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK)
  46
  47 /* fetch the pages addr resides in into pg and initialise sg with them */
  48 int __get_userbuf(uint8_t __user *addr, uint32_t len, int write,
  49                 int pgcount, struct page **pg, struct scatterlist *sg,
  50                 struct task_struct *task, struct mm_struct *mm)
  51 {
  52         int ret, pglen, i = 0;
  53         struct scatterlist *sgp;
  54
  55         down_write(&mm->mmap_sem);
  56         ret = get_user_pages(task, mm,
  57                         (unsigned long)addr, pgcount, write, 0, pg, NULL);
  58         up_write(&mm->mmap_sem);
  59         if (ret != pgcount)
  60                 return -EINVAL;
  61
  62         sg_init_table(sg, pgcount);
  63
  64         pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len);
  65         sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr));
  66
  67         len -= pglen;
  68         for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) {
  69                 pglen = min((uint32_t)PAGE_SIZE, len);
  70                 sg_set_page(sgp, pg[i++], pglen, 0);
  71                 len -= pglen;
  72         }
  73         sg_mark_end(sg_last(sg, pgcount));
  74         return 0;
  75 }
  76
  77 int adjust_sg_array(struct csession * ses, int pagecount)
  78 {
  79 struct scatterlist *sg;
  80 struct page **pages;
  81 int array_size;
  82
  83         for (array_size = ses->array_size; array_size < pagecount;
  84              array_size *= 2)
  85                 ;
  86
  87         dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n",
  88                         __func__, array_size);
  89         pages = krealloc(ses->pages, array_size * sizeof(struct page *),
  90                          GFP_KERNEL);
  91         if (unlikely(!pages))
  92                 return -ENOMEM;
  93         ses->pages = pages;
  94         sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist),
  95                       GFP_KERNEL);
  96         if (unlikely(!sg))
  97                 return -ENOMEM;
  98         ses->sg = sg;
  99         ses->array_size = array_size;
 100
 101         return 0;
 102 }
 103
 104 void release_user_pages(struct page **pg, int pagecount)
 105 {
 106         while (pagecount--) {
 107                 if (!PageReserved(pg[pagecount]))
 108                         SetPageDirty(pg[pagecount]);
 109                 page_cache_release(pg[pagecount]);
 110         }
 111 }
 112
 113 /* make src and dst available in scatterlists.
 114  * dst might be the same as src.
 115  */
 116 int get_userbuf(struct csession *ses, void* __user src, int src_len,
 117                 void* __user dst, int dst_len,
 118                 struct task_struct *task, struct mm_struct *mm,
 119                 struct scatterlist **src_sg,
 120                 struct scatterlist **dst_sg,
 121                 int *tot_pages)
 122 {
 123         int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1;
 124         int rc;
 125
 126         if (src == NULL)
 127                 return -EINVAL;
 128
 129         if (ses->alignmask && !IS_ALIGNED((unsigned long)src, ses->alignmask)) {
 130                 dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n",
 131                                 __func__, (unsigned long)src, ses->alignmask + 1);
 132         }
 133
 134         if (src == dst) {
 135                 /* dst == src */
 136                 src_len = max(src_len, dst_len);
 137                 dst_len = src_len;
 138         }
 139
 140         src_pagecount = PAGECOUNT(src, src_len);
 141         if (!ses->cdata.init) {         /* hashing only */
 142                 write_src = 0;
 143         } else if (src != dst) {        /* non-in-situ transformation */
 144                 if (dst == NULL)
 145                         return -EINVAL;
 146
 147                 dst_pagecount = PAGECOUNT(dst, dst_len);
 148                 write_src = 0;
 149
 150                 if (ses->alignmask && !IS_ALIGNED((unsigned long)dst, ses->alignmask)) {
 151                         dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n",
 152                                         __func__, (unsigned long)dst, ses->alignmask + 1);
 153                 }
 154         }
 155         (*tot_pages) = pagecount = src_pagecount + dst_pagecount;
 156
 157         if (pagecount > ses->array_size) {
 158                 rc = adjust_sg_array(ses, pagecount);
 159                 if (rc)
 160                         return rc;
 161         }
 162
 163         rc = __get_userbuf(src, src_len, write_src, src_pagecount,
 164                            ses->pages, ses->sg, task, mm);
 165         if (unlikely(rc)) {
 166                 dprintk(1, KERN_ERR,
 167                         "failed to get user pages for data input\n");
 168                 return -EINVAL;
 169         }
 170         (*src_sg) = (*dst_sg) = ses->sg;
 171
 172         if (!dst_pagecount)
 173                 return 0;
 174
 175         (*dst_sg) = ses->sg + src_pagecount;
 176
 177         rc = __get_userbuf(dst, dst_len, 1, dst_pagecount,
 178                            ses->pages + src_pagecount, *dst_sg,
 179                            task, mm);
 180         if (unlikely(rc)) {
 181                 dprintk(1, KERN_ERR,
 182                         "failed to get user pages for data output\n");
 183                 release_user_pages(ses->pages, src_pagecount);
 184                 return -EINVAL;
 185         }
 186         return 0;
 187 }