2 * Copyright (c) 2005-2008 Pawel Jakub Dawidek <pjd@FreeBSD.org>
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. Redistributions in binary form must reproduce the above copyright
11 * notice, this list of conditions and the following disclaimer in the
12 * documentation and/or other materials provided with the distribution.
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * $FreeBSD: src/sys/crypto/via/padlock.c,v 1.23 2009/02/05 19:30:28 imp Exp $
29 #include <sys/param.h>
30 #include <sys/systm.h>
31 #include <sys/kernel.h>
32 #include <sys/module.h>
34 #include <sys/spinlock.h>
35 #include <sys/spinlock2.h>
36 #include <sys/malloc.h>
37 #include <sys/libkern.h>
38 #if defined(__x86_64__) || defined(__i386__)
39 #include <machine/cpufunc.h>
40 #include <machine/cputypes.h>
41 #include <machine/md_var.h>
42 #include <machine/specialreg.h>
45 #include <opencrypto/cryptodev.h>
47 #include <crypto/via/padlock.h>
51 #include "cryptodev_if.h"
54 * Technical documentation about the PadLock engine can be found here:
56 * http://www.via.com.tw/en/downloads/whitepapers/initiatives/padlock/programming_guide.pdf
59 struct padlock_softc
{
62 TAILQ_HEAD(padlock_sessions_head
, padlock_session
) sc_sessions
;
63 struct spinlock sc_sessions_lock
;
66 static int padlock_newsession(device_t
, uint32_t *sidp
, struct cryptoini
*cri
);
67 static int padlock_freesession(device_t
, uint64_t tid
);
68 static void padlock_freesession_one(struct padlock_softc
*sc
,
69 struct padlock_session
*ses
, int locked
);
70 static int padlock_process(device_t
, struct cryptop
*crp
, int hint __unused
);
72 MALLOC_DEFINE(M_PADLOCK
, "padlock_data", "PadLock Data");
75 padlock_identify(driver_t
*drv
, device_t parent
)
77 /* NB: order 10 is so we get attached after h/w devices */
78 /* XXX: wouldn't bet about this BUS_ADD_CHILD correctness */
79 if (device_find_child(parent
, "padlock", -1) == NULL
&&
80 BUS_ADD_CHILD(parent
, parent
, 10, "padlock", -1) == 0)
81 panic("padlock: could not attach");
85 padlock_probe(device_t dev
)
89 #if defined(__x86_64__) || defined(__i386__)
90 /* If there is no AES support, we has nothing to do here. */
91 if (!(via_feature_xcrypt
& VIA_HAS_AES
)) {
92 device_printf(dev
, "No ACE support.\n");
95 strlcpy(capp
, "AES-CBC", sizeof(capp
));
97 strlcat(capp
, ",AES-EBC", sizeof(capp
));
98 strlcat(capp
, ",AES-CFB", sizeof(capp
));
99 strlcat(capp
, ",AES-OFB", sizeof(capp
));
101 if (via_feature_xcrypt
& VIA_HAS_SHA
) {
102 strlcat(capp
, ",SHA1", sizeof(capp
));
103 strlcat(capp
, ",SHA256", sizeof(capp
));
106 if (via_feature_xcrypt
& VIA_HAS_AESCTR
)
107 strlcat(capp
, ",AES-CTR", sizeof(capp
));
108 if (via_feature_xcrypt
& VIA_HAS_MM
)
109 strlcat(capp
, ",RSA", sizeof(capp
));
111 device_set_desc_copy(dev
, capp
);
119 padlock_attach(device_t dev
)
121 struct padlock_softc
*sc
= device_get_softc(dev
);
123 TAILQ_INIT(&sc
->sc_sessions
);
126 sc
->sc_cid
= crypto_get_driverid(dev
, CRYPTOCAP_F_HARDWARE
);
127 if (sc
->sc_cid
< 0) {
128 device_printf(dev
, "Could not get crypto driver id.\n");
132 spin_init(&sc
->sc_sessions_lock
);
133 crypto_register(sc
->sc_cid
, CRYPTO_AES_CBC
, 0, 0);
134 crypto_register(sc
->sc_cid
, CRYPTO_MD5_HMAC
, 0, 0);
135 crypto_register(sc
->sc_cid
, CRYPTO_SHA1_HMAC
, 0, 0);
136 crypto_register(sc
->sc_cid
, CRYPTO_RIPEMD160_HMAC
, 0, 0);
137 crypto_register(sc
->sc_cid
, CRYPTO_SHA2_256_HMAC
, 0, 0);
138 crypto_register(sc
->sc_cid
, CRYPTO_SHA2_384_HMAC
, 0, 0);
139 crypto_register(sc
->sc_cid
, CRYPTO_SHA2_512_HMAC
, 0, 0);
144 padlock_detach(device_t dev
)
146 struct padlock_softc
*sc
= device_get_softc(dev
);
147 struct padlock_session
*ses
;
149 spin_lock(&sc
->sc_sessions_lock
);
150 TAILQ_FOREACH(ses
, &sc
->sc_sessions
, ses_next
) {
152 spin_unlock(&sc
->sc_sessions_lock
);
154 "Cannot detach, sessions still active.\n");
158 while ((ses
= TAILQ_FIRST(&sc
->sc_sessions
)) != NULL
) {
159 TAILQ_REMOVE(&sc
->sc_sessions
, ses
, ses_next
);
160 kfree(ses
->ses_freeaddr
, M_PADLOCK
);
162 spin_unlock(&sc
->sc_sessions_lock
);
163 spin_uninit(&sc
->sc_sessions_lock
);
164 crypto_unregister_all(sc
->sc_cid
);
169 padlock_newsession(device_t dev
, uint32_t *sidp
, struct cryptoini
*cri
)
171 struct padlock_softc
*sc
= device_get_softc(dev
);
172 struct padlock_session
*ases
, *ses
= NULL
;
173 struct cryptoini
*encini
, *macini
;
176 if (sidp
== NULL
|| cri
== NULL
)
179 encini
= macini
= NULL
;
180 for (; cri
!= NULL
; cri
= cri
->cri_next
) {
181 switch (cri
->cri_alg
) {
182 case CRYPTO_NULL_HMAC
:
183 case CRYPTO_MD5_HMAC
:
184 case CRYPTO_SHA1_HMAC
:
185 case CRYPTO_RIPEMD160_HMAC
:
186 case CRYPTO_SHA2_256_HMAC
:
187 case CRYPTO_SHA2_384_HMAC
:
188 case CRYPTO_SHA2_512_HMAC
:
204 * We only support HMAC algorithms to be able to work with
205 * ipsec(4), so if we are asked only for authentication without
206 * encryption, don't pretend we can accellerate it.
212 * Let's look for a free session structure.
214 spin_lock(&sc
->sc_sessions_lock
);
216 * Free sessions goes first, so if first session is used, we need to
219 ses
= TAILQ_FIRST(&sc
->sc_sessions
);
220 if (ses
== NULL
|| ses
->ses_used
) {
221 ses
= kmalloc(sizeof(*ses
) + 16, M_PADLOCK
, M_NOWAIT
| M_ZERO
);
223 spin_unlock(&sc
->sc_sessions_lock
);
226 /* Check if 'ses' is 16-byte aligned. If not, align it. */
227 if (((uintptr_t)ses
& 0xf) != 0) {
228 ases
= PADLOCK_ALIGN(ses
);
229 ases
->ses_freeaddr
= ses
;
232 ses
->ses_freeaddr
= ses
;
234 ses
->ses_id
= sc
->sc_sid
++;
236 TAILQ_REMOVE(&sc
->sc_sessions
, ses
, ses_next
);
239 TAILQ_INSERT_TAIL(&sc
->sc_sessions
, ses
, ses_next
);
240 spin_unlock(&sc
->sc_sessions_lock
);
242 error
= padlock_cipher_setup(ses
, encini
);
244 padlock_freesession_one(sc
, ses
, 0);
248 if (macini
!= NULL
) {
249 error
= padlock_hash_setup(ses
, macini
);
251 padlock_freesession_one(sc
, ses
, 0);
261 padlock_freesession_one(struct padlock_softc
*sc
, struct padlock_session
*ses
,
264 uint32_t sid
= ses
->ses_id
;
265 void *freeaddr
= ses
->ses_freeaddr
;
268 spin_lock(&sc
->sc_sessions_lock
);
269 TAILQ_REMOVE(&sc
->sc_sessions
, ses
, ses_next
);
270 padlock_hash_free(ses
);
271 bzero(ses
, sizeof(*ses
));
274 ses
->ses_freeaddr
= freeaddr
;
275 TAILQ_INSERT_HEAD(&sc
->sc_sessions
, ses
, ses_next
);
277 spin_unlock(&sc
->sc_sessions_lock
);
281 padlock_freesession(device_t dev
, uint64_t tid
)
283 struct padlock_softc
*sc
= device_get_softc(dev
);
284 struct padlock_session
*ses
;
285 uint32_t sid
= ((uint32_t)tid
) & 0xffffffff;
287 spin_lock(&sc
->sc_sessions_lock
);
288 TAILQ_FOREACH_REVERSE(ses
, &sc
->sc_sessions
, padlock_sessions_head
,
290 if (ses
->ses_id
== sid
)
294 spin_unlock(&sc
->sc_sessions_lock
);
297 padlock_freesession_one(sc
, ses
, 1);
298 spin_unlock(&sc
->sc_sessions_lock
);
303 padlock_process(device_t dev
, struct cryptop
*crp
, int hint __unused
)
305 struct padlock_softc
*sc
= device_get_softc(dev
);
306 struct padlock_session
*ses
= NULL
;
307 struct cryptodesc
*crd
, *enccrd
, *maccrd
;
310 enccrd
= maccrd
= NULL
;
316 if (crp
->crp_callback
== NULL
|| crp
->crp_desc
== NULL
) {
321 for (crd
= crp
->crp_desc
; crd
!= NULL
; crd
= crd
->crd_next
) {
322 switch (crd
->crd_alg
) {
323 case CRYPTO_NULL_HMAC
:
324 case CRYPTO_MD5_HMAC
:
325 case CRYPTO_SHA1_HMAC
:
326 case CRYPTO_RIPEMD160_HMAC
:
327 case CRYPTO_SHA2_256_HMAC
:
328 case CRYPTO_SHA2_384_HMAC
:
329 case CRYPTO_SHA2_512_HMAC
:
330 if (maccrd
!= NULL
) {
337 if (enccrd
!= NULL
) {
347 if (enccrd
== NULL
|| (enccrd
->crd_len
% AES_BLOCK_LEN
) != 0) {
352 spin_lock(&sc
->sc_sessions_lock
); /* XXX: was rd lock */
353 TAILQ_FOREACH_REVERSE(ses
, &sc
->sc_sessions
, padlock_sessions_head
,
355 if (ses
->ses_id
== (crp
->crp_sid
& 0xffffffff))
358 spin_unlock(&sc
->sc_sessions_lock
); /* XXX: was rd lock */
364 /* Perform data authentication if requested before encryption. */
365 if (maccrd
!= NULL
&& maccrd
->crd_next
== enccrd
) {
366 error
= padlock_hash_process(ses
, maccrd
, crp
);
371 error
= padlock_cipher_process(ses
, enccrd
, crp
);
375 /* Perform data authentication if requested after encryption. */
376 if (maccrd
!= NULL
&& enccrd
->crd_next
== maccrd
) {
377 error
= padlock_hash_process(ses
, maccrd
, crp
);
385 * This code is not necessary, because contexts will be freed on next
386 * padlock_setup_mackey() call or at padlock_freesession() call.
388 if (ses
!= NULL
&& maccrd
!= NULL
&&
389 (maccrd
->crd_flags
& CRD_F_KEY_EXPLICIT
) != 0) {
390 padlock_free_ctx(ses
->ses_axf
, ses
->ses_ictx
);
391 padlock_free_ctx(ses
->ses_axf
, ses
->ses_octx
);
394 crp
->crp_etype
= error
;
399 static device_method_t padlock_methods
[] = {
400 DEVMETHOD(device_identify
, padlock_identify
),
401 DEVMETHOD(device_probe
, padlock_probe
),
402 DEVMETHOD(device_attach
, padlock_attach
),
403 DEVMETHOD(device_detach
, padlock_detach
),
405 DEVMETHOD(cryptodev_newsession
, padlock_newsession
),
406 DEVMETHOD(cryptodev_freesession
,padlock_freesession
),
407 DEVMETHOD(cryptodev_process
, padlock_process
),
412 static driver_t padlock_driver
= {
415 sizeof(struct padlock_softc
),
417 static devclass_t padlock_devclass
;
419 /* XXX where to attach */
420 DRIVER_MODULE(padlock
, nexus
, padlock_driver
, padlock_devclass
, 0, 0);
421 MODULE_VERSION(padlock
, 1);
422 MODULE_DEPEND(padlock
, crypto
, 1, 1, 1);