CMS_decrypt sporadically returns success when decrypting with wrong key

Given:

• Two pairs of private key / certificates (RSA): pkeyA+certA and pkeyB+certB.
• Message encrypted with certA.

Cases:

1. Try to decrypt with the wrong key/certificate pair pkeyB and certB. CMS_decrypt behaves as expected. No issues with this case.
2. Try to decrypt with the wrong key pkeyB and do not provide certificate (NULL). CMS_decrypt works unstable and returns 1 sometimes.
3. Try to decrypt with the wrong key pkeyB and certificate certA. CMS_decrypt produces unstable result, returns 1 sometimes.

CMS_decrypt is expected to return 0 when wrong key is provided, which doesn't happen sometimes. Function test_cms_decrypt in the snippet below invokes CSM_decrypt multiples times inside a cycle with the same input data and sometimes CMS_decrypt produces wrong result.

There is no issue when CMS_DEBUG_DECRYPT flag is set.

CMS_ContentInfo* create_cms(const std::string& message)
{
        BIO* msgBio = BIO_new(BIO_s_mem());
        BIO_write(msgBio, message.c_str(), static_cast<int>(message.size()));

        CMS_ContentInfo* cms = CMS_ContentInfo_new();

        if (!PEM_read_bio_CMS(msgBio, &cms, nullptr, nullptr))
        {
                BIO_vfree(msgBio);
                return nullptr;
        }

        BIO_vfree(msgBio);
        return cms;
}

std::string encrypt_message(const std::string& message, X509* cert)
{
        BIO* inBio = BIO_new(BIO_s_mem());
        BIO_write(inBio, message.c_str(), static_cast<int>(message.size()));

        STACK_OF(X509)* recipients = sk_X509_new_null();
        sk_X509_push(recipients, cert);

        CMS_ContentInfo* cms(
                CMS_encrypt(recipients, inBio, EVP_aes_256_cbc(), 0));

        sk_X509_free(recipients);

        if (!cms)
        {
                std::cout << "EncryptMessage() Failed to encrypt" << std::endl;
                CMS_ContentInfo_free(cms);
                BIO_vfree(inBio);
                return {};
        }

        BIO* outBio = BIO_new(BIO_s_mem());
        if (!PEM_write_bio_CMS_stream(outBio, cms, inBio, 0))
        {
                std::cout << "EncryptMessage() Failed to write cms" << std::endl;
                CMS_ContentInfo_free(cms);
                BIO_vfree(inBio);
                BIO_vfree(outBio);
                return {};
        }

        BUF_MEM* bio_buf;
        BIO_get_mem_ptr(outBio, &bio_buf);

        auto result =
                std::string(reinterpret_cast<const char*>(bio_buf->data), bio_buf->length);

        CMS_ContentInfo_free(cms);
        BIO_vfree(inBio);
        BIO_vfree(outBio);

        return result;
}

void test_cms_decrypt(CMS_ContentInfo* cms, EVP_PKEY* pkey, X509* cert)
{
        for (int i=0; i<1000; ++i)
        {
                BIO* outBio = BIO_new(BIO_s_mem());
                if (CMS_decrypt(cms, pkey, cert, NULL, outBio, 0) == 1) // no issues when CMS_DEBUG_DECRYPT is set
                {
                        std::cout << "Problem detected. This shouldn't happen! Iteration: " << i << std::endl;
                        BIO_vfree(outBio);
                        break;
                }
                BIO_vfree(outBio);
        }
}

int main(int argc, char* argv[])
{
        OpenSSL_add_all_algorithms();
        ERR_load_crypto_strings();

        const std::string message = "Hello world!";

        // Please add your own implementation here. Load cert/keys from file or generate them.
        EVP_PKEY* pkeyA;
        EVP_PKEY* pkeyB;
        X509* certA;
        X509* certB;

        const std::string encryptedMessage = encrypt_message(message, certA);
        CMS_ContentInfo* cms = create_cms(encryptedMessage);

        test_cms_decrypt(cms, pkeyB, certA);

        CMS_ContentInfo_free(cms);
        return 0;
}