openssl: Disable padding after initializing the cipher suite

according to OpenSSL documentation [1], EVP_CIPHER_CTX_set_padding()
should be called after EVP_EncryptInit_ex(), EVP_DecryptInit_ex(), or
EVP_CipherInit_ex(). Not doing this causes EVP_CIPHER_CTX_set_padding()
to return false on OpenSSL-3.0.0, resulting in the impossibility to
connect in many scenarios. Fix this changing the order of function calls
where needed.

[1] https://www.openssl.org/docs/man1.1.1/man3/EVP_CIPHER_CTX_set_padding.html

Reported-by: Vladimir Benes <vbenes@redhat.com>
Signed-off-by: Davide Caratti <davide.caratti@gmail.com>
This commit is contained in:
Davide Caratti 2021-08-17 10:58:53 +02:00 committed by Jouni Malinen
parent d265dd2d96
commit e2e9adc3d9

View file

@ -248,8 +248,8 @@ int rc4_skip(const u8 *key, size_t keylen, size_t skip,
ctx = EVP_CIPHER_CTX_new(); ctx = EVP_CIPHER_CTX_new();
if (!ctx || if (!ctx ||
!EVP_CIPHER_CTX_set_padding(ctx, 0) ||
!EVP_CipherInit_ex(ctx, EVP_rc4(), NULL, NULL, NULL, 1) || !EVP_CipherInit_ex(ctx, EVP_rc4(), NULL, NULL, NULL, 1) ||
!EVP_CIPHER_CTX_set_padding(ctx, 0) ||
!EVP_CIPHER_CTX_set_key_length(ctx, keylen) || !EVP_CIPHER_CTX_set_key_length(ctx, keylen) ||
!EVP_CipherInit_ex(ctx, NULL, NULL, key, NULL, 1)) !EVP_CipherInit_ex(ctx, NULL, NULL, key, NULL, 1))
goto out; goto out;
@ -709,8 +709,8 @@ struct crypto_cipher * crypto_cipher_init(enum crypto_cipher_alg alg,
} }
if (!(ctx->enc = EVP_CIPHER_CTX_new()) || if (!(ctx->enc = EVP_CIPHER_CTX_new()) ||
!EVP_CIPHER_CTX_set_padding(ctx->enc, 0) ||
!EVP_EncryptInit_ex(ctx->enc, cipher, NULL, NULL, NULL) || !EVP_EncryptInit_ex(ctx->enc, cipher, NULL, NULL, NULL) ||
!EVP_CIPHER_CTX_set_padding(ctx->enc, 0) ||
!EVP_CIPHER_CTX_set_key_length(ctx->enc, key_len) || !EVP_CIPHER_CTX_set_key_length(ctx->enc, key_len) ||
!EVP_EncryptInit_ex(ctx->enc, NULL, NULL, key, iv)) { !EVP_EncryptInit_ex(ctx->enc, NULL, NULL, key, iv)) {
if (ctx->enc) if (ctx->enc)
@ -720,8 +720,8 @@ struct crypto_cipher * crypto_cipher_init(enum crypto_cipher_alg alg,
} }
if (!(ctx->dec = EVP_CIPHER_CTX_new()) || if (!(ctx->dec = EVP_CIPHER_CTX_new()) ||
!EVP_CIPHER_CTX_set_padding(ctx->dec, 0) ||
!EVP_DecryptInit_ex(ctx->dec, cipher, NULL, NULL, NULL) || !EVP_DecryptInit_ex(ctx->dec, cipher, NULL, NULL, NULL) ||
!EVP_CIPHER_CTX_set_padding(ctx->dec, 0) ||
!EVP_CIPHER_CTX_set_key_length(ctx->dec, key_len) || !EVP_CIPHER_CTX_set_key_length(ctx->dec, key_len) ||
!EVP_DecryptInit_ex(ctx->dec, NULL, NULL, key, iv)) { !EVP_DecryptInit_ex(ctx->dec, NULL, NULL, key, iv)) {
EVP_CIPHER_CTX_free(ctx->enc); EVP_CIPHER_CTX_free(ctx->enc);