From 8108c1602ef1286e0250e3a67b90ff07ccb5c69e Mon Sep 17 00:00:00 2001 From: Nikos Mavrogiannopoulos Date: Thu, 1 Dec 2011 21:22:13 +0100 Subject: [PATCH] zero copy functions separated from rest. --- Makefile | 2 +- authenc.c | 126 +++++++++++++++++++++++++++++---------------------- cryptodev_int.h | 13 ------ ioctl.c | 71 +---------------------------- main.c | 136 ++++++++++++++++++++++++++++---------------------------- zc.c | 110 +++++++++++++++++++++++++++++++++++++++++++++ zc.h | 12 +++++ 7 files changed, 265 insertions(+), 205 deletions(-) create mode 100644 zc.c create mode 100644 zc.h diff --git a/Makefile b/Makefile index a01bab1..01f6d66 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ KBUILD_CFLAGS += -I$(src) KERNEL_DIR = /lib/modules/$(shell uname -r)/build VERSION = 1.0 -cryptodev-objs = ioctl.o main.o cryptlib.o authenc.o +cryptodev-objs = ioctl.o main.o cryptlib.o authenc.o zc.o obj-m += cryptodev.o diff --git a/authenc.c b/authenc.c index 5975159..dab3963 100644 --- a/authenc.c +++ b/authenc.c @@ -40,29 +40,28 @@ #include #include #include "cryptodev_int.h" +#include "zc.h" #include "version.h" -/* make cop->src and cop->dst available in scatterlists */ -static int get_userbuf_aead(struct csession *ses, struct kernel_crypt_auth_op *kcaop, - struct scatterlist **auth_sg, struct scatterlist **dst_sg, +/* make caop->dst available in scatterlist. + * (caop->src is assumed to be equal to caop->dst) + */ +static int get_userbuf_tls(struct csession *ses, struct kernel_crypt_auth_op *kcaop, + struct scatterlist **dst_sg, int *tot_pages) { - int dst_pagecount = 0, pagecount; - int auth_pagecount = 0; + int pagecount = 0; struct crypt_auth_op *caop = &kcaop->caop; int rc; - if (caop->dst == NULL && caop->auth_src == NULL) + if (caop->dst == NULL) return -EINVAL; if (ses->alignmask) { if (!IS_ALIGNED((unsigned long)caop->dst, ses->alignmask)) dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n", __func__, (unsigned long)caop->dst, ses->alignmask + 1); - if (!IS_ALIGNED((unsigned long)caop->auth_src, ses->alignmask)) - dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n", - __func__, (unsigned long)caop->auth_src, ses->alignmask + 1); } if (kcaop->dst_len == 0) { @@ -70,40 +69,23 @@ static int get_userbuf_aead(struct csession *ses, struct kernel_crypt_auth_op *k return -EINVAL; } - if (caop->auth_len > 0) - auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len); - - dst_pagecount = PAGECOUNT(caop->dst, kcaop->dst_len); + pagecount = PAGECOUNT(caop->dst, kcaop->dst_len); - (*tot_pages) = pagecount = auth_pagecount + dst_pagecount; + (*tot_pages) = pagecount; rc = adjust_sg_array(ses, pagecount); if (rc) return rc; - if (auth_pagecount > 0) { - rc = __get_userbuf(caop->auth_src, caop->auth_len, 0, auth_pagecount, - ses->pages, ses->sg, kcaop->task, kcaop->mm); - if (unlikely(rc)) { - dprintk(1, KERN_ERR, - "failed to get user pages for data input\n"); - return -EINVAL; - } - (*auth_sg) = ses->sg; - (*dst_sg) = ses->sg + auth_pagecount; - } else { - (*auth_sg) = NULL; - (*dst_sg) = ses->sg; - } - - rc = __get_userbuf(caop->dst, kcaop->dst_len, 1, dst_pagecount, - ses->pages + auth_pagecount, *dst_sg, kcaop->task, kcaop->mm); + rc = __get_userbuf(caop->dst, kcaop->dst_len, 1, pagecount, + ses->pages, ses->sg, kcaop->task, kcaop->mm); if (unlikely(rc)) { - release_user_pages(ses->pages, auth_pagecount); dprintk(1, KERN_ERR, "failed to get user pages for data input\n"); return -EINVAL; } + + (*dst_sg) = ses->sg; return 0; } @@ -163,6 +145,14 @@ static int sg_copy(struct scatterlist *sg_from, struct scatterlist *sg_to, int l return 0; } +#define MAX_SRTP_AUTH_DATA_DIFF 256 + +/* Makes caop->auth_src available as scatterlist. + * It also provides a pointer to caop->dst, which however, + * is assumed to be within the caop->auth_src buffer. If not + * (if their difference exceeds MAX_SRTP_AUTH_DATA_DIFF) it + * returns error. + */ static int get_userbuf_srtp(struct csession *ses, struct kernel_crypt_auth_op *kcaop, struct scatterlist **auth_sg, struct scatterlist **dst_sg, int *tot_pages) @@ -194,7 +184,7 @@ static int get_userbuf_srtp(struct csession *ses, struct kernel_crypt_auth_op *k auth_pagecount = PAGECOUNT(caop->auth_src, caop->auth_len); diff = (int)(caop->src - caop->auth_src); - if (diff > PAGE_SIZE || diff < 0) { + if (diff > MAX_SRTP_AUTH_DATA_DIFF || diff < 0) { dprintk(1, KERN_WARNING, "auth_src must overlap with src (diff: %d).\n", diff); return -EINVAL; } @@ -273,15 +263,15 @@ static int fill_kcaop_from_caop(struct kernel_crypt_auth_op *kcaop, struct fcryp return -EINVAL; } - if (caop->src != caop->dst) { - dprintk(2, KERN_ERR, - "Non-inplace encryption and decryption is not efficient\n"); + if (caop->flags & COP_FLAG_AEAD_TLS_TYPE || caop->flags & COP_FLAG_AEAD_SRTP_TYPE) { + if (caop->src != caop->dst) { + dprintk(2, KERN_ERR, + "Non-inplace encryption and decryption is not efficient\n"); - rc = copy_from_user_to_user( caop->dst, caop->src, caop->len); - if (rc < 0) - goto out_unlock; - - + rc = copy_from_user_to_user( caop->dst, caop->src, caop->len); + if (rc < 0) + goto out_unlock; + } } if (caop->tag_len == 0) @@ -644,18 +634,9 @@ __crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcao { struct scatterlist *dst_sg, *auth_sg; struct crypt_auth_op *caop = &kcaop->caop; - int ret = 0, pagecount; - - if (caop->flags & COP_FLAG_AEAD_TLS_TYPE) { - ret = get_userbuf_aead(ses_ptr, kcaop, &auth_sg, &dst_sg, &pagecount); - if (unlikely(ret)) { - dprintk(1, KERN_ERR, "Error getting user pages.\n"); - return ret; - } + int ret = 0, pagecount = 0; - ret = tls_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len, - dst_sg, caop->len); - } else if (caop->flags & COP_FLAG_AEAD_SRTP_TYPE) { + if (caop->flags & COP_FLAG_AEAD_SRTP_TYPE) { ret = get_userbuf_srtp(ses_ptr, kcaop, &auth_sg, &dst_sg, &pagecount); if (unlikely(ret)) { dprintk(1, KERN_ERR, "Error getting user pages.\n"); @@ -665,8 +646,45 @@ __crypto_auth_run_zc(struct csession *ses_ptr, struct kernel_crypt_auth_op *kcao ret = srtp_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len, dst_sg, caop->len); } else { - dprintk(1, KERN_ERR, "Unsupported flag for authenc\n"); - return -EINVAL; + unsigned char* buf; + struct scatterlist tmp; + + if (unlikely(caop->auth_len > PAGE_SIZE)) + return -EINVAL; + + buf = (char *)__get_free_page(GFP_KERNEL); + + if (unlikely(!buf)) + return -ENOMEM; + + if (caop->auth_len > 0) { + if (unlikely(copy_from_user(buf, caop->auth_src, caop->auth_len))) { + ret = -EFAULT; + goto fail; + } + + sg_init_one(&tmp, buf, caop->auth_len); + auth_sg = &tmp; + } else { + auth_sg = NULL; + } + + if (caop->flags & COP_FLAG_AEAD_TLS_TYPE) { + ret = get_userbuf_tls(ses_ptr, kcaop, &dst_sg, &pagecount); + if (unlikely(ret)) { + dprintk(1, KERN_ERR, "Error getting user pages.\n"); + goto fail; + } + + ret = tls_auth_n_crypt(ses_ptr, kcaop, auth_sg, caop->auth_len, + dst_sg, caop->len); + } else { + dprintk(1, KERN_ERR, "Unsupported flag for authenc\n"); + return -EINVAL; + } + +fail: + free_page((unsigned long)buf); } release_user_pages(ses_ptr->pages, pagecount); diff --git a/cryptodev_int.h b/cryptodev_int.h index 634f195..e0523db 100644 --- a/cryptodev_int.h +++ b/cryptodev_int.h @@ -25,19 +25,6 @@ extern int cryptodev_verbosity; -/* For zero copy */ -int __get_userbuf(uint8_t __user *addr, uint32_t len, int write, - int pgcount, struct page **pg, struct scatterlist *sg, - struct task_struct *task, struct mm_struct *mm); -void release_user_pages(struct page **pg, int pagecount); - -/* last page - first page + 1 */ -#define PAGECOUNT(buf, buflen) \ - ((((unsigned long)(buf + buflen - 1) & PAGE_MASK) >> PAGE_SHIFT) - \ - (((unsigned long) buf & PAGE_MASK) >> PAGE_SHIFT) + 1) - -#define DEFAULT_PREALLOC_PAGES 32 - struct cipher_data { int init; /* 0 uninitialized */ int blocksize; diff --git a/ioctl.c b/ioctl.c index aa8ad51..67f797b 100644 --- a/ioctl.c +++ b/ioctl.c @@ -45,6 +45,7 @@ #include #include #include "cryptodev_int.h" +#include "zc.h" #include "version.h" MODULE_AUTHOR("Nikos Mavrogiannopoulos "); @@ -403,76 +404,6 @@ crypto_get_session_by_sid(struct fcrypt *fcr, uint32_t sid) return retval; } -void release_user_pages(struct page **pg, int pagecount) -{ - while (pagecount--) { - if (!PageReserved(pg[pagecount])) - SetPageDirty(pg[pagecount]); - page_cache_release(pg[pagecount]); - } -} - -/* offset of buf in it's first page */ -#define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK) - -/* fetch the pages addr resides in into pg and initialise sg with them */ -int __get_userbuf(uint8_t __user *addr, uint32_t len, int write, - int pgcount, struct page **pg, struct scatterlist *sg, - struct task_struct *task, struct mm_struct *mm) -{ - int ret, pglen, i = 0; - struct scatterlist *sgp; - - down_write(&mm->mmap_sem); - ret = get_user_pages(task, mm, - (unsigned long)addr, pgcount, write, 0, pg, NULL); - up_write(&mm->mmap_sem); - if (ret != pgcount) - return -EINVAL; - - sg_init_table(sg, pgcount); - - pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len); - sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr)); - - len -= pglen; - for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) { - pglen = min((uint32_t)PAGE_SIZE, len); - sg_set_page(sgp, pg[i++], pglen, 0); - len -= pglen; - } - sg_mark_end(sg_last(sg, pgcount)); - return 0; -} - -int adjust_sg_array(struct csession * ses, int pagecount) -{ -struct scatterlist *sg; -struct page **pages; -int array_size; - - for (array_size = ses->array_size; array_size < pagecount; - array_size *= 2) - ; - - dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n", - __func__, array_size); - pages = krealloc(ses->pages, array_size * sizeof(struct page *), - GFP_KERNEL); - if (unlikely(!pages)) - return -ENOMEM; - ses->pages = pages; - sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist), - GFP_KERNEL); - if (unlikely(!sg)) - return -ENOMEM; - ses->sg = sg; - ses->array_size = array_size; - - return 0; -} - - static void cryptask_routine(struct work_struct *work) { struct crypt_priv *pcr = container_of(work, struct crypt_priv, cryptask); diff --git a/main.c b/main.c index b79282d..f1167e5 100644 --- a/main.c +++ b/main.c @@ -45,79 +45,13 @@ #include #include #include "cryptodev_int.h" +#include "zc.h" #include "version.h" /* This file contains the traditional operations of encryption * and hashing of /dev/crypto. */ -/* make cop->src and cop->dst available in scatterlists */ -static int get_userbuf(struct csession *ses, struct kernel_crypt_op *kcop, - struct scatterlist **src_sg, struct scatterlist **dst_sg, - int *tot_pages) -{ - int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1; - struct crypt_op *cop = &kcop->cop; - int rc; - - if (cop->src == NULL) - return -EINVAL; - - if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->src, ses->alignmask)) { - dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n", - __func__, (unsigned long)cop->src, ses->alignmask + 1); - } - - src_pagecount = PAGECOUNT(cop->src, cop->len); - if (!ses->cdata.init) { /* hashing only */ - write_src = 0; - } else if (cop->src != cop->dst) { /* non-in-situ transformation */ - if (cop->dst == NULL) - return -EINVAL; - - dst_pagecount = PAGECOUNT(cop->dst, cop->len); - write_src = 0; - - if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->dst, ses->alignmask)) { - dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n", - __func__, (unsigned long)cop->dst, ses->alignmask + 1); - } - - } - (*tot_pages) = pagecount = src_pagecount + dst_pagecount; - - if (pagecount > ses->array_size) { - rc = adjust_sg_array(ses, pagecount); - if (rc) - return rc; - } - - rc = __get_userbuf(cop->src, cop->len, write_src, src_pagecount, - ses->pages, ses->sg, kcop->task, kcop->mm); - if (unlikely(rc)) { - dprintk(1, KERN_ERR, - "failed to get user pages for data input\n"); - return -EINVAL; - } - (*src_sg) = (*dst_sg) = ses->sg; - - if (!dst_pagecount) - return 0; - - (*dst_sg) = ses->sg + src_pagecount; - - rc = __get_userbuf(cop->dst, cop->len, 1, dst_pagecount, - ses->pages + src_pagecount, *dst_sg, - kcop->task, kcop->mm); - if (unlikely(rc)) { - dprintk(1, KERN_ERR, - "failed to get user pages for data output\n"); - release_user_pages(ses->pages, src_pagecount); - return -EINVAL; - } - return 0; -} - static int hash_n_crypt(struct csession *ses_ptr, struct crypt_op *cop, struct scatterlist *src_sg, struct scatterlist *dst_sg, @@ -217,6 +151,74 @@ __crypto_run_std(struct csession *ses_ptr, struct crypt_op *cop) return ret; } +/* make cop->src and cop->dst available in scatterlists */ +static int get_userbuf(struct csession *ses, struct kernel_crypt_op *kcop, + struct scatterlist **src_sg, struct scatterlist **dst_sg, + int *tot_pages) +{ + int src_pagecount, dst_pagecount = 0, pagecount, write_src = 1; + struct crypt_op *cop = &kcop->cop; + int rc; + + if (cop->src == NULL) + return -EINVAL; + + if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->src, ses->alignmask)) { + dprintk(2, KERN_WARNING, "%s: careful - source address %lx is not %d byte aligned\n", + __func__, (unsigned long)cop->src, ses->alignmask + 1); + } + + src_pagecount = PAGECOUNT(cop->src, cop->len); + if (!ses->cdata.init) { /* hashing only */ + write_src = 0; + } else if (cop->src != cop->dst) { /* non-in-situ transformation */ + if (cop->dst == NULL) + return -EINVAL; + + dst_pagecount = PAGECOUNT(cop->dst, cop->len); + write_src = 0; + + if (ses->alignmask && !IS_ALIGNED((unsigned long)cop->dst, ses->alignmask)) { + dprintk(2, KERN_WARNING, "%s: careful - destination address %lx is not %d byte aligned\n", + __func__, (unsigned long)cop->dst, ses->alignmask + 1); + } + + } + (*tot_pages) = pagecount = src_pagecount + dst_pagecount; + + if (pagecount > ses->array_size) { + rc = adjust_sg_array(ses, pagecount); + if (rc) + return rc; + } + + rc = __get_userbuf(cop->src, cop->len, write_src, src_pagecount, + ses->pages, ses->sg, kcop->task, kcop->mm); + if (unlikely(rc)) { + dprintk(1, KERN_ERR, + "failed to get user pages for data input\n"); + return -EINVAL; + } + (*src_sg) = (*dst_sg) = ses->sg; + + if (!dst_pagecount) + return 0; + + (*dst_sg) = ses->sg + src_pagecount; + + rc = __get_userbuf(cop->dst, cop->len, 1, dst_pagecount, + ses->pages + src_pagecount, *dst_sg, + kcop->task, kcop->mm); + if (unlikely(rc)) { + dprintk(1, KERN_ERR, + "failed to get user pages for data output\n"); + release_user_pages(ses->pages, src_pagecount); + return -EINVAL; + } + return 0; +} + + /* This is the main crypto function - zero-copy edition */ static int __crypto_run_zc(struct csession *ses_ptr, struct kernel_crypt_op *kcop) diff --git a/zc.c b/zc.c new file mode 100644 index 0000000..906dd06 --- /dev/null +++ b/zc.c @@ -0,0 +1,110 @@ +/* + * Driver for /dev/crypto device (aka CryptoDev) + * + * Copyright (c) 2009-2011 Nikos Mavrogiannopoulos + * Copyright (c) 2010 Phil Sutter + * + * This file is part of linux cryptodev. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version 2 + * of the License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA + * 02110-1301, USA. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "cryptodev_int.h" +#include "version.h" + +/* Helper functions to assist zero copy. + * This needs to be redesigned and moved out of the session. --nmav + */ + +/* offset of buf in it's first page */ +#define PAGEOFFSET(buf) ((unsigned long)buf & ~PAGE_MASK) + +/* fetch the pages addr resides in into pg and initialise sg with them */ +int __get_userbuf(uint8_t __user *addr, uint32_t len, int write, + int pgcount, struct page **pg, struct scatterlist *sg, + struct task_struct *task, struct mm_struct *mm) +{ + int ret, pglen, i = 0; + struct scatterlist *sgp; + + down_write(&mm->mmap_sem); + ret = get_user_pages(task, mm, + (unsigned long)addr, pgcount, write, 0, pg, NULL); + up_write(&mm->mmap_sem); + if (ret != pgcount) + return -EINVAL; + + sg_init_table(sg, pgcount); + + pglen = min((ptrdiff_t)(PAGE_SIZE - PAGEOFFSET(addr)), (ptrdiff_t)len); + sg_set_page(sg, pg[i++], pglen, PAGEOFFSET(addr)); + + len -= pglen; + for (sgp = sg_next(sg); len; sgp = sg_next(sgp)) { + pglen = min((uint32_t)PAGE_SIZE, len); + sg_set_page(sgp, pg[i++], pglen, 0); + len -= pglen; + } + sg_mark_end(sg_last(sg, pgcount)); + return 0; +} + +int adjust_sg_array(struct csession * ses, int pagecount) +{ +struct scatterlist *sg; +struct page **pages; +int array_size; + + for (array_size = ses->array_size; array_size < pagecount; + array_size *= 2) + ; + + dprintk(2, KERN_DEBUG, "%s: reallocating to %d elements\n", + __func__, array_size); + pages = krealloc(ses->pages, array_size * sizeof(struct page *), + GFP_KERNEL); + if (unlikely(!pages)) + return -ENOMEM; + ses->pages = pages; + sg = krealloc(ses->sg, array_size * sizeof(struct scatterlist), + GFP_KERNEL); + if (unlikely(!sg)) + return -ENOMEM; + ses->sg = sg; + ses->array_size = array_size; + + return 0; +} + +void release_user_pages(struct page **pg, int pagecount) +{ + while (pagecount--) { + if (!PageReserved(pg[pagecount])) + SetPageDirty(pg[pagecount]); + page_cache_release(pg[pagecount]); + } +} diff --git a/zc.h b/zc.h new file mode 100644 index 0000000..e9f4cd4 --- /dev/null +++ b/zc.h @@ -0,0 +1,12 @@ +/* For zero copy */ +int __get_userbuf(uint8_t __user *addr, uint32_t len, int write, + int pgcount, struct page **pg, struct scatterlist *sg, + struct task_struct *task, struct mm_struct *mm); +void release_user_pages(struct page **pg, int pagecount); + +/* last page - first page + 1 */ +#define PAGECOUNT(buf, buflen) \ + ((((unsigned long)(buf + buflen - 1) & PAGE_MASK) >> PAGE_SHIFT) - \ + (((unsigned long) buf & PAGE_MASK) >> PAGE_SHIFT) + 1) + +#define DEFAULT_PREALLOC_PAGES 32 -- 2.11.4.GIT