From: Daniel Jordan <daniel.m.jordan@oracle.com> mainline inclusion from mainline-v5.4-rc1 commit e6ce0e0807e90d38a2cefa524ac253d7a85c3f2f category: bugfix bugzilla: https://gitee.com/openeuler/kernel/issues/I7RZMT CVE: NA Reference: https://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git/commit/?i... -------------------------------- padata_do_parallel currently returns -EINVAL if the callback CPU isn't in the callback cpumask. pcrypt tries to prevent this situation by keeping its own callback cpumask in sync with padata's and checks that the callback CPU it passes to padata is valid. Make padata handle this instead. padata_do_parallel now takes a pointer to the callback CPU and updates it for the caller if an alternate CPU is used. Overall behavior in terms of which callback CPUs are chosen stays the same. Prepares for removal of the padata cpumask notifier in pcrypt, which will fix a lockdep complaint about nested acquisition of the CPU hotplug lock later in the series. Signed-off-by: Daniel Jordan <daniel.m.jordan@oracle.com> Acked-by: Steffen Klassert <steffen.klassert@secunet.com> Cc: Herbert Xu <herbert@gondor.apana.org.au> Cc: Lai Jiangshan <jiangshanlai@gmail.com> Cc: Peter Zijlstra <peterz@infradead.org> Cc: Tejun Heo <tj@kernel.org> Cc: linux-crypto@vger.kernel.org Cc: linux-kernel@vger.kernel.org Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> Conflicts: crypto/pcrypt.c include/linux/padata.h kernel/padata.c [Context conflict.] Signed-off-by: Yi Yang <yiyang13@huawei.com> --- crypto/pcrypt.c | 33 ++------------------------------- include/linux/padata.h | 2 +- kernel/padata.c | 27 ++++++++++++++++++++------- 3 files changed, 23 insertions(+), 39 deletions(-) diff --git a/crypto/pcrypt.c b/crypto/pcrypt.c index 898a71c7386b..4542b48b5afd 100644 --- a/crypto/pcrypt.c +++ b/crypto/pcrypt.c @@ -70,35 +70,6 @@ struct pcrypt_aead_ctx { unsigned int cb_cpu; }; -static int pcrypt_do_parallel(struct padata_priv *padata, unsigned int *cb_cpu, - struct padata_pcrypt *pcrypt) -{ - unsigned int cpu_index, cpu, i; - struct pcrypt_cpumask *cpumask; - - cpu = *cb_cpu; - - rcu_read_lock_bh(); - cpumask = rcu_dereference_bh(pcrypt->cb_cpumask); - if (cpumask_test_cpu(cpu, cpumask->mask)) - goto out; - - if (!cpumask_weight(cpumask->mask)) - goto out; - - cpu_index = cpu % cpumask_weight(cpumask->mask); - - cpu = cpumask_first(cpumask->mask); - for (i = 0; i < cpu_index; i++) - cpu = cpumask_next(cpu, cpumask->mask); - - *cb_cpu = cpu; - -out: - rcu_read_unlock_bh(); - return padata_do_parallel(pcrypt->pinst, padata, cpu); -} - static int pcrypt_aead_setkey(struct crypto_aead *parent, const u8 *key, unsigned int keylen) { @@ -172,7 +143,7 @@ static int pcrypt_aead_encrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_ad(creq, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pencrypt); + err = padata_do_parallel(pencrypt.pinst, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; if (err == -EBUSY) { @@ -220,7 +191,7 @@ static int pcrypt_aead_decrypt(struct aead_request *req) req->cryptlen, req->iv); aead_request_set_ad(creq, req->assoclen); - err = pcrypt_do_parallel(padata, &ctx->cb_cpu, &pdecrypt); + err = padata_do_parallel(pdecrypt.pinst, padata, &ctx->cb_cpu); if (!err) return -EINPROGRESS; if (err == -EBUSY) { diff --git a/include/linux/padata.h b/include/linux/padata.h index d803397a28f7..8d2267926a64 100644 --- a/include/linux/padata.h +++ b/include/linux/padata.h @@ -167,7 +167,7 @@ extern struct padata_instance *padata_alloc_possible( struct workqueue_struct *wq); extern void padata_free(struct padata_instance *pinst); extern int padata_do_parallel(struct padata_instance *pinst, - struct padata_priv *padata, int cb_cpu); + struct padata_priv *padata, int *cb_cpu); extern void padata_do_serial(struct padata_priv *padata); extern int padata_set_cpumask(struct padata_instance *pinst, int cpumask_type, cpumask_var_t cpumask); diff --git a/kernel/padata.c b/kernel/padata.c index 848e40e3581b..3518c2e60965 100644 --- a/kernel/padata.c +++ b/kernel/padata.c @@ -96,17 +96,19 @@ static void padata_parallel_worker(struct work_struct *parallel_work) * * @pinst: padata instance * @padata: object to be parallelized - * @cb_cpu: cpu the serialization callback function will run on, - * must be in the serial cpumask of padata(i.e. cpumask.cbcpu). + * @cb_cpu: pointer to the CPU that the serialization callback function should + * run on. If it's not in the serial cpumask of @pinst + * (i.e. cpumask.cbcpu), this function selects a fallback CPU and if + * none found, returns -EINVAL. * * The parallelization callback function will run with BHs off. * Note: Every object which is parallelized by padata_do_parallel * must be seen by padata_do_serial. */ int padata_do_parallel(struct padata_instance *pinst, - struct padata_priv *padata, int cb_cpu) + struct padata_priv *padata, int *cb_cpu) { - int target_cpu, err; + int i, cpu, cpu_index, target_cpu, err; struct padata_parallel_queue *queue; struct parallel_data *pd; @@ -118,8 +120,19 @@ int padata_do_parallel(struct padata_instance *pinst, if (!(pinst->flags & PADATA_INIT) || pinst->flags & PADATA_INVALID) goto out; - if (!cpumask_test_cpu(cb_cpu, pd->cpumask.cbcpu)) - goto out; + if (!cpumask_test_cpu(*cb_cpu, pd->cpumask.cbcpu)) { + if (!cpumask_weight(pd->cpumask.cbcpu)) + goto out; + + /* Select an alternate fallback CPU and notify the caller. */ + cpu_index = *cb_cpu % cpumask_weight(pd->cpumask.cbcpu); + + cpu = cpumask_first(pd->cpumask.cbcpu); + for (i = 0; i < cpu_index; i++) + cpu = cpumask_next(cpu, pd->cpumask.cbcpu); + + *cb_cpu = cpu; + } err = -EBUSY; if ((pinst->flags & PADATA_RESET)) @@ -131,7 +144,7 @@ int padata_do_parallel(struct padata_instance *pinst, err = 0; atomic_inc(&pd->refcnt); padata->pd = pd; - padata->cb_cpu = cb_cpu; + padata->cb_cpu = *cb_cpu; target_cpu = padata_cpu_hash(pd); padata->cpu = target_cpu; -- 2.25.1