D10384.1790899298.diff
No OneTemporary
Actions

Size

20 KB

Referenced Files

None

Subscribers

None

D10384.1790899298.diff
View Options

	Index: opencrypto/crypto.c
	===================================================================
	--- opencrypto/crypto.c
	+++ opencrypto/crypto.c
	@@ -134,31 +134,70 @@
	* have one per-queue but having one simplifies handling of block/unblock
	* operations.
	*/
	-static int crp_sleep = 0;
	-static TAILQ_HEAD(,cryptop) crp_q; /* request queues */
	-static TAILQ_HEAD(,cryptkop) crp_kq;
	-static struct mtx crypto_q_mtx;
	-#define CRYPTO_Q_LOCK() mtx_lock(&crypto_q_mtx)
	-#define CRYPTO_Q_UNLOCK() mtx_unlock(&crypto_q_mtx)
	-
	-/*
	- * There are two queues for processing completed crypto requests; one
	- * for the symmetric and one for the asymmetric ops. We only need one
	- * but have two to avoid type futzing (cryptop vs. cryptkop). A single
	- * mutex is used to lock access to both queues. Note that this lock
	- * must be separate from the lock on request queues to insure driver
	- * callbacks don't generate lock order reversals.
	- */
	-static TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */
	-static TAILQ_HEAD(,cryptkop) crp_ret_kq;
	-static struct mtx crypto_ret_q_mtx;
	-#define CRYPTO_RETQ_LOCK() mtx_lock(&crypto_ret_q_mtx)
	-#define CRYPTO_RETQ_UNLOCK() mtx_unlock(&crypto_ret_q_mtx)
	-#define CRYPTO_RETQ_EMPTY() (TAILQ_EMPTY(&crp_ret_q) && TAILQ_EMPTY(&crp_ret_kq))
	+struct crypto_worker {
	+ TAILQ_HEAD(,cryptop) crp_q; /* request queues */
	+ TAILQ_HEAD(,cryptkop) crp_kq;
	+
	+ struct mtx crypto_q_mtx;
	+
	+ int crp_sleep;
	+ struct proc *cryptoproc;
	+} __aligned(CACHE_LINE_SIZE);
	+static struct crypto_worker *crypto_workers = NULL;
	+
	+#define CRYPTO_W(i) (&crypto_workers[i])
	+#define CRYPTO_W_ID(w) ((int)(((uintptr_t)w - (uintptr_t)crypto_workers) / sizeof(struct crypto_worker)))
	+
	+#define CRYPTO_W_FOREACH(w) \
	+ for (w = crypto_workers; w < crypto_workers + crypto_workers_num; ++w)
	+#define CRYPTO_W_LOCK(w) mtx_lock(&(w)->crypto_q_mtx)
	+#define CRYPTO_W_UNLOCK(w) mtx_unlock(&(w)->crypto_q_mtx)
	+
	+#define CRYPTO_W_LOCKALL() do { \
	+ struct crypto_worker *_w; \
	+ CRYPTO_W_FOREACH(_w) \
	+ CRYPTO_W_LOCK(_w); \
	+} while (0)
	+
	+#define CRYPTO_W_UNLOCKALL() do { \
	+ struct crypto_worker *_w; \
	+ CRYPTO_W_FOREACH(_w) \
	+ CRYPTO_W_UNLOCK(_w); \
	+} while (0)
	+
	+struct crypto_ret_worker {
	+ /*
	+ * There are two queues for processing completed crypto requests; one
	+ * for the symmetric and one for the asymmetric ops. We only need one
	+ * but have two to avoid type futzing (cryptop vs. cryptkop). A single
	+ * mutex is used to lock access to both queues. Note that this lock
	+ * must be separate from the lock on request queues to insure driver
	+ * callbacks don't generate lock order reversals.
	+ */
	+ TAILQ_HEAD(,cryptop) crp_ret_q; /* callback queues */
	+ TAILQ_HEAD(,cryptkop) crp_ret_kq;
	+ struct mtx crypto_ret_q_mtx;
	+
	+ struct proc *cryptoretproc;
	+} __aligned(CACHE_LINE_SIZE);
	+static struct crypto_ret_worker *crypto_ret_workers = NULL;
	+
	+#define CRYPTO_RETW_FOREACH(w) \
	+ for (w = crypto_ret_workers; w < crypto_ret_workers + crypto_workers_num; ++w)
	+
	+#define CRYPTO_RETW(i) (&crypto_ret_workers[i])
	+#define CRYPTO_RETW_ID(w) ((int)(((uintptr_t)(w) - (uintptr_t)crypto_ret_workers) / sizeof(struct crypto_ret_worker)))
	+
	+#define CRYPTO_RETQ_LOCK(w) mtx_lock(&w->crypto_ret_q_mtx)
	+#define CRYPTO_RETQ_UNLOCK(w) mtx_unlock(&w->crypto_ret_q_mtx)
	+#define CRYPTO_RETQ_EMPTY(w) (TAILQ_EMPTY(&w->crp_ret_q) && TAILQ_EMPTY(&w->crp_ret_kq))

	static uma_zone_t cryptop_zone;
	static uma_zone_t cryptodesc_zone;

	+static int crypto_workers_num = 1;
	+TUNABLE_INT("kern.crypto_workers_num", &crypto_workers_num);
	+
	int crypto_userasymcrypto = 1; /* userland may do asym crypto reqs */
	SYSCTL_INT(_kern, OID_AUTO, userasymcrypto, CTLFLAG_RW,
	&crypto_userasymcrypto, 0,
	@@ -167,13 +206,23 @@
	SYSCTL_INT(_kern, OID_AUTO, cryptodevallowsoft, CTLFLAG_RW,
	&crypto_devallowsoft, 0,
	"Enable/disable use of software crypto by /dev/crypto");
	+int crypto_max_requests = 256;
	+SYSCTL_INT(_kern, OID_AUTO, crypto_max_requests, CTLFLAG_RW,
	+ &crypto_max_requests, 0,
	+ "Max number of pending crypto requests");
	+int crypto_nb_requests = 0;
	+SYSCTL_INT(_kern, OID_AUTO, crypto_nb_requests, CTLFLAG_RW,
	+ &crypto_nb_requests, 0,
	+ "Current number of crypto requests");
	+int crypto_dropped_requests = 0;
	+SYSCTL_INT(_kern, OID_AUTO, crypto_dropped_requests, CTLFLAG_RW,
	+ &crypto_dropped_requests, 0,
	+ "Dropped requests due to limit reached");

	MALLOC_DEFINE(M_CRYPTO_DATA, "crypto", "crypto session records");

	-static void crypto_proc(void);
	-static struct proc *cryptoproc;
	-static void crypto_ret_proc(void);
	-static struct proc *cryptoretproc;
	+static void crypto_proc(struct crypto_worker *worker);
	+static void crypto_ret_proc(struct crypto_ret_worker *worker);
	static void crypto_destroy(void);
	static int crypto_invoke(struct cryptocap cap, struct cryptop crp, int hint);
	static int crypto_kinvoke(struct cryptkop *krp, int flags);
	@@ -220,21 +269,36 @@
	}

	static int
	+select_crypto_worker_id(const struct cryptop *crp)
	+{
	+ /* Each worker is assigned a session */
	+ if (crypto_workers_num > 1)
	+ return (crp->crp_sid % crypto_workers_num);
	+
	+ return 0;
	+}
	+
	+static int
	+select_kcrypto_worker_id(struct cryptkop *krp)
	+{
	+ if (crypto_workers_num > 1)
	+ return krp->krp_crid % crypto_workers_num;
	+
	+ return 0;
	+}
	+
	+static int
	crypto_init(void)
	{
	+ struct crypto_ret_worker *ret_worker;
	+ struct crypto_worker *worker;
	int error;

	+ crypto_nb_requests = 0;
	+
	mtx_init(&crypto_drivers_mtx, "crypto", "crypto driver table",
	MTX_DEF\|MTX_QUIET);

	- TAILQ_INIT(&crp_q);
	- TAILQ_INIT(&crp_kq);
	- mtx_init(&crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
	-
	- TAILQ_INIT(&crp_ret_q);
	- TAILQ_INIT(&crp_ret_kq);
	- mtx_init(&crypto_ret_q_mtx, "crypto", "crypto return queues", MTX_DEF);
	-
	cryptop_zone = uma_zcreate("cryptop", sizeof (struct cryptop),
	0, 0, 0, 0,
	UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
	@@ -256,24 +320,63 @@
	goto bad;
	}

	- error = kproc_create((void ()(void )) crypto_proc, NULL,
	- &cryptoproc, 0, 0, "crypto");
	- if (error) {
	- printf("crypto_init: cannot start crypto thread; error %d",
	- error);
	+ if (crypto_workers_num < 1)
	+ crypto_workers_num = 1;
	+
	+ crypto_workers = malloc(crypto_workers_num * sizeof(struct crypto_worker),
	+ M_CRYPTO_DATA, M_NOWAIT\|M_ZERO);
	+ if (crypto_workers == NULL) {
	+ error = ENOMEM;
	+ printf("crypto_init: cannot allocate workers\n");
	goto bad;
	}

	- error = kproc_create((void ()(void )) crypto_ret_proc, NULL,
	- &cryptoretproc, 0, 0, "crypto returns");
	- if (error) {
	- printf("crypto_init: cannot start cryptoret thread; error %d",
	- error);
	+ if (bootverbose)
	+ printf("crypto_init: instanciating %d worker(s)\n", crypto_workers_num);
	+
	+ CRYPTO_W_FOREACH(worker) {
	+ TAILQ_INIT(&worker->crp_q);
	+ TAILQ_INIT(&worker->crp_kq);
	+ worker->crp_sleep = 0;
	+
	+ mtx_init(&worker->crypto_q_mtx, "crypto", "crypto op queues", MTX_DEF);
	+
	+ error = kproc_create((void ()(void )) crypto_proc, worker,
	+ &worker->cryptoproc, 0, 0, "crypto %d", CRYPTO_W_ID(worker));
	+ if (error) {
	+ printf("crypto_init: cannot start crypto thread %d; error %d",
	+ CRYPTO_W_ID(worker), error);
	+ goto bad;
	+ }
	+ }
	+
	+ crypto_ret_workers = malloc(crypto_workers_num * sizeof(struct crypto_ret_worker),
	+ M_CRYPTO_DATA, M_NOWAIT\|M_ZERO);
	+ if (crypto_ret_workers == NULL) {
	+ error = ENOMEM;
	+ printf("crypto_init: cannot allocate ret workers\n");
	goto bad;
	}

	keybuf_init();

	+ CRYPTO_RETW_FOREACH(ret_worker) {
	+ TAILQ_INIT(&ret_worker->crp_ret_q);
	+ TAILQ_INIT(&ret_worker->crp_ret_kq);
	+
	+ mtx_init(&ret_worker->crypto_ret_q_mtx, "crypto",
	+ "crypto return queues", MTX_DEF);
	+
	+ error = kproc_create((void ()(void )) crypto_ret_proc, ret_worker,
	+ &ret_worker->cryptoretproc, 0, 0, "crypto returns %d",
	+ CRYPTO_RETW_ID(ret_worker));
	+ if (error) {
	+ printf("crypto_init: cannot start crypto ret thread %d; error %d",
	+ CRYPTO_RETW_ID(ret_worker), error);
	+ goto bad;
	+ }
	+ }
	+
	return 0;
	bad:
	crypto_destroy();
	@@ -292,7 +395,7 @@
	{
	struct proc *p;

	- mtx_assert(&crypto_drivers_mtx, MA_OWNED);
	+ CRYPTO_DRIVER_ASSERT();
	p = *pp;
	*pp = NULL;
	if (p) {
	@@ -308,12 +411,17 @@
	static void
	crypto_destroy(void)
	{
	+ struct crypto_ret_worker *ret_worker;
	+ struct crypto_worker *worker;
	+
	/*
	* Terminate any crypto threads.
	*/
	CRYPTO_DRIVER_LOCK();
	- crypto_terminate(&cryptoproc, &crp_q);
	- crypto_terminate(&cryptoretproc, &crp_ret_q);
	+ CRYPTO_W_FOREACH(worker)
	+ crypto_terminate(&worker->cryptoproc, &worker->crp_q);
	+ CRYPTO_RETW_FOREACH(ret_worker)
	+ crypto_terminate(&ret_worker->cryptoretproc, &ret_worker->crp_ret_q);
	CRYPTO_DRIVER_UNLOCK();

	/* XXX flush queues??? */
	@@ -328,8 +436,15 @@
	uma_zdestroy(cryptodesc_zone);
	if (cryptop_zone != NULL)
	uma_zdestroy(cryptop_zone);
	- mtx_destroy(&crypto_q_mtx);
	- mtx_destroy(&crypto_ret_q_mtx);
	+
	+ CRYPTO_W_FOREACH(worker)
	+ mtx_destroy(&worker->crypto_q_mtx);
	+ free(crypto_workers, M_CRYPTO_DATA);
	+
	+ CRYPTO_RETW_FOREACH(ret_worker)
	+ mtx_destroy(&ret_worker->crypto_ret_q_mtx);
	+ free(crypto_ret_workers, M_CRYPTO_DATA);
	+
	mtx_destroy(&crypto_drivers_mtx);
	}

	@@ -792,21 +907,25 @@
	crypto_unblock(u_int32_t driverid, int what)
	{
	struct cryptocap *cap;
	+ struct crypto_worker *worker;
	int err;

	- CRYPTO_Q_LOCK();
	+ CRYPTO_W_LOCKALL();
	cap = crypto_checkdriver(driverid);
	if (cap != NULL) {
	if (what & CRYPTO_SYMQ)
	cap->cc_qblocked = 0;
	if (what & CRYPTO_ASYMQ)
	cap->cc_kqblocked = 0;
	- if (crp_sleep)
	- wakeup_one(&crp_q);
	+ CRYPTO_W_FOREACH(worker) {
	+ if (worker->crp_sleep)
	+ wakeup_one(&worker->crp_q);
	+ }
	+
	err = 0;
	} else
	err = EINVAL;
	- CRYPTO_Q_UNLOCK();
	+ CRYPTO_W_UNLOCKALL();

	return err;
	}
	@@ -818,10 +937,11 @@
	crypto_dispatch(struct cryptop *crp)
	{
	struct cryptocap *cap;
	+ struct crypto_worker *worker;
	u_int32_t hid;
	int result;

	- cryptostats.cs_ops++;
	+ atomic_add_acq_int(&cryptostats.cs_ops, 1);

	#ifdef CRYPTO_TIMING
	if (crypto_timing)
	@@ -849,11 +969,14 @@
	*/
	}
	}
	- CRYPTO_Q_LOCK();
	- TAILQ_INSERT_TAIL(&crp_q, crp, crp_next);
	- if (crp_sleep)
	- wakeup_one(&crp_q);
	- CRYPTO_Q_UNLOCK();
	+
	+ worker = CRYPTO_W(select_crypto_worker_id(crp));
	+ CRYPTO_W_LOCK(worker);
	+ TAILQ_INSERT_TAIL(&worker->crp_q, crp, crp_next);
	+ if (worker->crp_sleep)
	+ wakeup_one(&worker->crp_q);
	+ CRYPTO_W_UNLOCK(worker);
	+
	return 0;
	}

	@@ -870,11 +993,15 @@

	error = crypto_kinvoke(krp, krp->krp_crid);
	if (error == ERESTART) {
	- CRYPTO_Q_LOCK();
	- TAILQ_INSERT_TAIL(&crp_kq, krp, krp_next);
	- if (crp_sleep)
	- wakeup_one(&crp_q);
	- CRYPTO_Q_UNLOCK();
	+ struct crypto_worker *worker;
	+
	+ worker = CRYPTO_W(select_kcrypto_worker_id(krp));
	+
	+ CRYPTO_W_LOCK(worker);
	+ TAILQ_INSERT_TAIL(&worker->crp_kq, krp, krp_next);
	+ if (worker->crp_sleep)
	+ wakeup_one(&worker->crp_q);
	+ CRYPTO_W_UNLOCK(worker);
	error = 0;
	}
	return error;
	@@ -1094,30 +1221,10 @@
	if (crp == NULL)
	return;

	-#ifdef DIAGNOSTIC
	- {
	- struct cryptop *crp2;
	-
	- CRYPTO_Q_LOCK();
	- TAILQ_FOREACH(crp2, &crp_q, crp_next) {
	- KASSERT(crp2 != crp,
	- ("Freeing cryptop from the crypto queue (%p).",
	- crp));
	- }
	- CRYPTO_Q_UNLOCK();
	- CRYPTO_RETQ_LOCK();
	- TAILQ_FOREACH(crp2, &crp_ret_q, crp_next) {
	- KASSERT(crp2 != crp,
	- ("Freeing cryptop from the return queue (%p).",
	- crp));
	- }
	- CRYPTO_RETQ_UNLOCK();
	- }
	-#endif
	-
	while ((crd = crp->crp_desc) != NULL) {
	crp->crp_desc = crd->crd_next;
	uma_zfree(cryptodesc_zone, crd);
	+ atomic_add_acq_int(&crypto_nb_requests, -1);
	}
	uma_zfree(cryptop_zone, crp);
	}
	@@ -1131,6 +1238,11 @@
	struct cryptodesc *crd;
	struct cryptop *crp;

	+ if (crypto_nb_requests > crypto_max_requests) {
	+ crypto_dropped_requests++;
	+ return NULL;
	+ }
	+
	crp = uma_zalloc(cryptop_zone, M_NOWAIT\|M_ZERO);
	if (crp != NULL) {
	while (num--) {
	@@ -1142,6 +1254,7 @@

	crd->crd_next = crp->crp_desc;
	crp->crp_desc = crd;
	+ atomic_add_acq_int(&crypto_nb_requests, 1);
	}
	}
	return crp;
	@@ -1192,14 +1305,18 @@
	#endif
	crp->crp_callback(crp);
	} else {
	+// struct cryptop *tmp;
	+ struct crypto_ret_worker *ret_worker;
	+
	+ ret_worker = CRYPTO_RETW(select_crypto_worker_id(crp));
	/*
	* Normal case; queue the callback for the thread.
	*/
	- CRYPTO_RETQ_LOCK();
	- if (CRYPTO_RETQ_EMPTY())
	- wakeup_one(&crp_ret_q); /* shared wait channel */
	- TAILQ_INSERT_TAIL(&crp_ret_q, crp, crp_next);
	- CRYPTO_RETQ_UNLOCK();
	+ CRYPTO_RETQ_LOCK(ret_worker);
	+ if (CRYPTO_RETQ_EMPTY(ret_worker))
	+ wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
	+ TAILQ_INSERT_TAIL(&ret_worker->crp_ret_q, crp, crp_next);
	+ CRYPTO_RETQ_UNLOCK(ret_worker);
	}
	}

	@@ -1209,6 +1326,7 @@
	void
	crypto_kdone(struct cryptkop *krp)
	{
	+ struct crypto_ret_worker *ret_worker;
	struct cryptocap *cap;

	if (krp->krp_status != 0)
	@@ -1223,11 +1341,14 @@
	crypto_remove(cap);
	}
	CRYPTO_DRIVER_UNLOCK();
	- CRYPTO_RETQ_LOCK();
	- if (CRYPTO_RETQ_EMPTY())
	- wakeup_one(&crp_ret_q); /* shared wait channel */
	- TAILQ_INSERT_TAIL(&crp_ret_kq, krp, krp_next);
	- CRYPTO_RETQ_UNLOCK();
	+
	+ ret_worker = CRYPTO_RETW(select_kcrypto_worker_id(krp));
	+
	+ CRYPTO_RETQ_LOCK(ret_worker);
	+ if (CRYPTO_RETQ_EMPTY(ret_worker))
	+ wakeup_one(&ret_worker->crp_ret_q); /* shared wait channel */
	+ TAILQ_INSERT_TAIL(&ret_worker->crp_ret_kq, krp, krp_next);
	+ CRYPTO_RETQ_UNLOCK(ret_worker);
	}

	int
	@@ -1274,7 +1395,7 @@
	* Crypto thread, dispatches crypto requests.
	*/
	static void
	-crypto_proc(void)
	+crypto_proc(struct crypto_worker *worker)
	{
	struct cryptop crp, submit;
	struct cryptkop *krp;
	@@ -1286,7 +1407,7 @@
	fpu_kern_thread(FPU_KERN_NORMAL);
	#endif

	- CRYPTO_Q_LOCK();
	+ CRYPTO_W_LOCK(worker);
	for (;;) {
	/*
	* Find the first element in the queue that can be
	@@ -1295,7 +1416,7 @@
	*/
	submit = NULL;
	hint = 0;
	- TAILQ_FOREACH(crp, &crp_q, crp_next) {
	+ TAILQ_FOREACH(crp, &worker->crp_q, crp_next) {
	hid = CRYPTO_SESID2HID(crp->crp_sid);
	cap = crypto_checkdriver(hid);
	/*
	@@ -1332,7 +1453,7 @@
	}
	}
	if (submit != NULL) {
	- TAILQ_REMOVE(&crp_q, submit, crp_next);
	+ TAILQ_REMOVE(&worker->crp_q, submit, crp_next);
	hid = CRYPTO_SESID2HID(submit->crp_sid);
	cap = crypto_checkdriver(hid);
	KASSERT(cap != NULL, ("%s:%u Driver disappeared.",
	@@ -1350,13 +1471,13 @@
	*/
	/* XXX validate sid again? */
	crypto_drivers[CRYPTO_SESID2HID(submit->crp_sid)].cc_qblocked = 1;
	- TAILQ_INSERT_HEAD(&crp_q, submit, crp_next);
	- cryptostats.cs_blocks++;
	+ TAILQ_INSERT_HEAD(&worker->crp_q, submit, crp_next);
	+ atomic_add_acq_int(&cryptostats.cs_blocks, 1);
	}
	}

	/* As above, but for key ops */
	- TAILQ_FOREACH(krp, &crp_kq, krp_next) {
	+ TAILQ_FOREACH(krp, &worker->crp_kq, krp_next) {
	cap = crypto_checkdriver(krp->krp_hid);
	if (cap == NULL \|\| cap->cc_dev == NULL) {
	/*
	@@ -1376,7 +1497,7 @@
	break;
	}
	if (krp != NULL) {
	- TAILQ_REMOVE(&crp_kq, krp, krp_next);
	+ TAILQ_REMOVE(&worker->crp_kq, krp, krp_next);
	result = crypto_kinvoke(krp, krp->krp_hid);
	if (result == ERESTART) {
	/*
	@@ -1390,8 +1511,8 @@
	*/
	/* XXX validate sid again? */
	crypto_drivers[krp->krp_hid].cc_kqblocked = 1;
	- TAILQ_INSERT_HEAD(&crp_kq, krp, krp_next);
	- cryptostats.cs_kblocks++;
	+ TAILQ_INSERT_HEAD(&worker->crp_kq, krp, krp_next);
	+ atomic_add_acq_int(&cryptostats.cs_kblocks, 1);
	}
	}

	@@ -1408,17 +1529,17 @@
	* out of order if dispatched to different devices
	* and some become blocked while others do not.
	*/
	- crp_sleep = 1;
	- msleep(&crp_q, &crypto_q_mtx, PWAIT, "crypto_wait", 0);
	- crp_sleep = 0;
	- if (cryptoproc == NULL)
	+ worker->crp_sleep = 1;
	+ msleep(&worker->crp_q, &worker->crypto_q_mtx, PWAIT, "crypto_wait", 0);
	+ worker->crp_sleep = 0;
	+ if (worker->cryptoproc == NULL)
	break;
	- cryptostats.cs_intrs++;
	+ atomic_add_acq_int(&cryptostats.cs_intrs, 1);
	}
	}
	- CRYPTO_Q_UNLOCK();
	+ CRYPTO_W_UNLOCK(worker);

	- crypto_finis(&crp_q);
	+ crypto_finis(&worker->crp_q);
	}

	/*
	@@ -1427,24 +1548,24 @@
	* callbacks typically are expensive and would slow interrupt handling.
	*/
	static void
	-crypto_ret_proc(void)
	+crypto_ret_proc(struct crypto_ret_worker *worker)
	{
	struct cryptop *crpt;
	struct cryptkop *krpt;

	- CRYPTO_RETQ_LOCK();
	+ CRYPTO_RETQ_LOCK(worker);
	for (;;) {
	/* Harvest return q's for completed ops */
	- crpt = TAILQ_FIRST(&crp_ret_q);
	+ crpt = TAILQ_FIRST(&worker->crp_ret_q);
	if (crpt != NULL)
	- TAILQ_REMOVE(&crp_ret_q, crpt, crp_next);
	+ TAILQ_REMOVE(&worker->crp_ret_q, crpt, crp_next);

	- krpt = TAILQ_FIRST(&crp_ret_kq);
	+ krpt = TAILQ_FIRST(&worker->crp_ret_kq);
	if (krpt != NULL)
	- TAILQ_REMOVE(&crp_ret_kq, krpt, krp_next);
	+ TAILQ_REMOVE(&worker->crp_ret_kq, krpt, krp_next);

	if (crpt != NULL \|\| krpt != NULL) {
	- CRYPTO_RETQ_UNLOCK();
	+ CRYPTO_RETQ_UNLOCK(worker);
	/*
	* Run callbacks unlocked.
	*/
	@@ -1466,22 +1587,22 @@
	}
	if (krpt != NULL)
	krpt->krp_callback(krpt);
	- CRYPTO_RETQ_LOCK();
	+ CRYPTO_RETQ_LOCK(worker);
	} else {
	/*
	* Nothing more to be processed. Sleep until we're
	* woken because there are more returns to process.
	*/
	- msleep(&crp_ret_q, &crypto_ret_q_mtx, PWAIT,
	+ msleep(&worker->crp_ret_q, &worker->crypto_ret_q_mtx, PWAIT,
	"crypto_ret_wait", 0);
	- if (cryptoretproc == NULL)
	+ if (worker->cryptoretproc == NULL)
	break;
	cryptostats.cs_rets++;
	}
	}
	- CRYPTO_RETQ_UNLOCK();
	+ CRYPTO_RETQ_UNLOCK(worker);

	- crypto_finis(&crp_ret_q);
	+ crypto_finis(&worker->crp_ret_q);
	}

	#ifdef DDB
	@@ -1516,6 +1637,8 @@
	DB_SHOW_COMMAND(crypto, db_show_crypto)
	{
	struct cryptop *crp;
	+ struct crypto_ret_worker *ret_worker;
	+ struct crypto_worker *worker;

	db_show_drivers();
	db_printf("\n");
	@@ -1523,61 +1646,75 @@
	db_printf("%4s %8s %4s %4s %4s %4s %8s %8s\n",
	"HID", "Caps", "Ilen", "Olen", "Etype", "Flags",
	"Desc", "Callback");
	- TAILQ_FOREACH(crp, &crp_q, crp_next) {
	- db_printf("%4u %08x %4u %4u %4u %04x %8p %8p\n"
	- , (int) CRYPTO_SESID2HID(crp->crp_sid)
	- , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
	- , crp->crp_ilen, crp->crp_olen
	- , crp->crp_etype
	- , crp->crp_flags
	- , crp->crp_desc
	- , crp->crp_callback
	- );
	- }
	- if (!TAILQ_EMPTY(&crp_ret_q)) {
	- db_printf("\n%4s %4s %4s %8s\n",
	- "HID", "Etype", "Flags", "Callback");
	- TAILQ_FOREACH(crp, &crp_ret_q, crp_next) {
	- db_printf("%4u %4u %04x %8p\n"
	+ CRYPTO_W_FOREACH(worker) {
	+ TAILQ_FOREACH(crp, &worker->crp_q, crp_next) {
	+ db_printf("%4u %4u %08x %4u %4u %4u %04x %8p %8p\n"
	+ , CRYPTO_W_ID(worker)
	, (int) CRYPTO_SESID2HID(crp->crp_sid)
	+ , (int) CRYPTO_SESID2CAPS(crp->crp_sid)
	+ , crp->crp_ilen, crp->crp_olen
	, crp->crp_etype
	, crp->crp_flags
	+ , crp->crp_desc
	, crp->crp_callback
	);
	}
	}
	+ CRYPTO_RETW_FOREACH(ret_worker) {
	+ if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
	+ db_printf("\n%7s %4s %4s %4s %8s\n",
	+ "Worker", "HID", "Etype", "Flags", "Callback");
	+ TAILQ_FOREACH(crp, &ret_worker->crp_ret_q, crp_next) {
	+ db_printf("%4u %4u %4u %04x %8p\n"
	+ , CRYPTO_RETW_ID(ret_worker)
	+ , (int) CRYPTO_SESID2HID(crp->crp_sid)
	+ , crp->crp_etype
	+ , crp->crp_flags
	+ , crp->crp_callback
	+ );
	+ }
	+ }
	+ }
	}

	DB_SHOW_COMMAND(kcrypto, db_show_kcrypto)
	{
	struct cryptkop *krp;
	+ struct crypto_ret_worker *ret_worker;
	+ struct crypto_worker *worker;

	db_show_drivers();
	db_printf("\n");

	- db_printf("%4s %5s %4s %4s %8s %4s %8s\n",
	- "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
	- TAILQ_FOREACH(krp, &crp_kq, krp_next) {
	- db_printf("%4u %5u %4u %4u %08x %4u %8p\n"
	- , krp->krp_op
	- , krp->krp_status
	- , krp->krp_iparams, krp->krp_oparams
	- , krp->krp_crid, krp->krp_hid
	- , krp->krp_callback
	- );
	- }
	- if (!TAILQ_EMPTY(&crp_ret_q)) {
	- db_printf("%4s %5s %8s %4s %8s\n",
	- "Op", "Status", "CRID", "HID", "Callback");
	- TAILQ_FOREACH(krp, &crp_ret_kq, krp_next) {
	- db_printf("%4u %5u %08x %4u %8p\n"
	+ db_printf("%7s %4s %5s %4s %4s %8s %4s %8s\n",
	+ "Worker", "Op", "Status", "#IP", "#OP", "CRID", "HID", "Callback");
	+ CRYPTO_W_FOREACH(worker) {
	+ TAILQ_FOREACH(krp, &worker->crp_kq, krp_next) {
	+ db_printf("%4u %4u %5u %4u %4u %08x %4u %8p\n"
	+ , CRYPTO_W_ID(worker)
	, krp->krp_op
	, krp->krp_status
	+ , krp->krp_iparams, krp->krp_oparams
	, krp->krp_crid, krp->krp_hid
	, krp->krp_callback
	);
	}
	}
	+ CRYPTO_RETW_FOREACH(ret_worker) {
	+ if (!TAILQ_EMPTY(&ret_worker->crp_ret_q)) {
	+ db_printf("%11s %4s %5s %8s %4s %8s\n",
	+ "Ret Worker", "Op", "Status", "CRID", "HID", "Callback");
	+ TAILQ_FOREACH(krp, &ret_worker->crp_ret_kq, krp_next) {
	+ db_printf("%4u %4u %5u %08x %4u %8p\n"
	+ , CRYPTO_RETW_ID(ret_worker)
	+ , krp->krp_op
	+ , krp->krp_status
	+ , krp->krp_crid, krp->krp_hid
	+ , krp->krp_callback
	+ );
	+ }
	+ }
	+ }
	}
	#endif

File Metadata

Mime Type: text/plain
Expires: Fri, Oct 2, 12:01 AM (5 h, 26 m)
Storage Engine: blob
Storage Format: Raw Data
Storage Handle: 30081000
Default Alt Text: D10384.1790899298.diff (20 KB)

D10384.1790899298.diffNo OneTemporaryActions

D10384.1790899298.diffView Options

File Metadata

Event Timeline

D10384.1790899298.diff
No OneTemporary
Actions

D10384.1790899298.diff
View Options