From: Guus Sliepen <guus@tinc-vpn.org>
Date: Wed, 22 Jun 2016 14:32:05 +0000 (+0200)
Subject: Ensure compatibility with OpenSSL 1.1.0.
X-Git-Tag: release-1.1pre15~68
X-Git-Url: https://tinc-vpn.org/git/browse?a=commitdiff_plain;h=323c17e232539f3f06e7cebc664ab48f60127e0e;p=tinc

Ensure compatibility with OpenSSL 1.1.0.
---

diff --git a/m4/openssl.m4 b/m4/openssl.m4
index 26cbcbe8..6bb33cf5 100644
--- a/m4/openssl.m4
+++ b/m4/openssl.m4
@@ -45,7 +45,7 @@ AC_DEFUN([tinc_OPENSSL],
     [AC_MSG_ERROR([LibreSSL/OpenSSL libraries not found.])]
   )
 
-  AC_CHECK_FUNCS([RAND_status EVP_EncryptInit_ex], ,
+  AC_CHECK_FUNCS([RAND_bytes EVP_EncryptInit_ex EVP_CIPHER_CTX_new], ,
     [AC_MSG_ERROR([Missing LibreSSL/OpenSSL functionality, make sure you have installed the latest version.]); break],
   )
 
@@ -53,4 +53,6 @@ AC_DEFUN([tinc_OPENSSL],
     [AC_MSG_ERROR([Missing LibreSSL/OpenSSL functionality, make sure you have installed the latest version.]); break],
     [#include <openssl/evp.h>]
   )
+
+  AC_CHECK_FUNCS([BN_GENCB_new ERR_remove_state RSA_set0_key], , , [#include <openssl/rsa.h>])
 ])
diff --git a/src/openssl/cipher.c b/src/openssl/cipher.c
index 04aee27e..ae9640f0 100644
--- a/src/openssl/cipher.c
+++ b/src/openssl/cipher.c
@@ -28,14 +28,16 @@
 #include "../xalloc.h"
 
 struct cipher {
-	EVP_CIPHER_CTX ctx;
+	EVP_CIPHER_CTX *ctx;
 	const EVP_CIPHER *cipher;
 };
 
 static cipher_t *cipher_open(const EVP_CIPHER *evp_cipher) {
 	cipher_t *cipher = xzalloc(sizeof *cipher);
 	cipher->cipher = evp_cipher;
-	EVP_CIPHER_CTX_init(&cipher->ctx);
+	cipher->ctx = EVP_CIPHER_CTX_new();
+	if(!cipher->ctx)
+		abort();
 
 	return cipher;
 }
@@ -68,7 +70,7 @@ void cipher_close(cipher_t *cipher) {
 	if(!cipher)
 		return;
 
-	EVP_CIPHER_CTX_cleanup(&cipher->ctx);
+	EVP_CIPHER_CTX_free(cipher->ctx);
 	free(cipher);
 }
 
@@ -76,23 +78,23 @@ size_t cipher_keylength(const cipher_t *cipher) {
 	if(!cipher || !cipher->cipher)
 		return 0;
 
-	return cipher->cipher->key_len + cipher->cipher->iv_len;
+	return EVP_CIPHER_key_length(cipher->cipher) + EVP_CIPHER_iv_length(cipher->cipher);
 }
 
 size_t cipher_blocksize(const cipher_t *cipher) {
 	if(!cipher || !cipher->cipher)
 		return 1;
 
-	return cipher->cipher->block_size;
+	return EVP_CIPHER_block_size(cipher->cipher);
 }
 
 bool cipher_set_key(cipher_t *cipher, void *key, bool encrypt) {
 	bool result;
 
 	if(encrypt)
-		result = EVP_EncryptInit_ex(&cipher->ctx, cipher->cipher, NULL, (unsigned char *)key, (unsigned char *)key + cipher->cipher->key_len);
+		result = EVP_EncryptInit_ex(cipher->ctx, cipher->cipher, NULL, (unsigned char *)key, (unsigned char *)key + EVP_CIPHER_key_length(cipher->cipher));
 	else
-		result = EVP_DecryptInit_ex(&cipher->ctx, cipher->cipher, NULL, (unsigned char *)key, (unsigned char *)key + cipher->cipher->key_len);
+		result = EVP_DecryptInit_ex(cipher->ctx, cipher->cipher, NULL, (unsigned char *)key, (unsigned char *)key + EVP_CIPHER_key_length(cipher->cipher));
 
 	if(result)
 		return true;
@@ -105,9 +107,9 @@ bool cipher_set_key_from_rsa(cipher_t *cipher, void *key, size_t len, bool encry
 	bool result;
 
 	if(encrypt)
-		result = EVP_EncryptInit_ex(&cipher->ctx, cipher->cipher, NULL, (unsigned char *)key + len - cipher->cipher->key_len, (unsigned char *)key + len - cipher->cipher->iv_len - cipher->cipher->key_len);
+		result = EVP_EncryptInit_ex(cipher->ctx, cipher->cipher, NULL, (unsigned char *)key + len - EVP_CIPHER_key_length(cipher->cipher), (unsigned char *)key + len - EVP_CIPHER_iv_length(cipher->cipher) - EVP_CIPHER_key_length(cipher->cipher));
 	else
-		result = EVP_DecryptInit_ex(&cipher->ctx, cipher->cipher, NULL, (unsigned char *)key + len - cipher->cipher->key_len, (unsigned char *)key + len - cipher->cipher->iv_len - cipher->cipher->key_len);
+		result = EVP_DecryptInit_ex(cipher->ctx, cipher->cipher, NULL, (unsigned char *)key + len - EVP_CIPHER_key_length(cipher->cipher), (unsigned char *)key + len - EVP_CIPHER_iv_length(cipher->cipher) - EVP_CIPHER_key_length(cipher->cipher));
 
 	if(result)
 		return true;
@@ -119,15 +121,15 @@ bool cipher_set_key_from_rsa(cipher_t *cipher, void *key, size_t len, bool encry
 bool cipher_encrypt(cipher_t *cipher, const void *indata, size_t inlen, void *outdata, size_t *outlen, bool oneshot) {
 	if(oneshot) {
 		int len, pad;
-		if(EVP_EncryptInit_ex(&cipher->ctx, NULL, NULL, NULL, NULL)
-				&& EVP_EncryptUpdate(&cipher->ctx, (unsigned char *)outdata, &len, indata, inlen)
-				&& EVP_EncryptFinal(&cipher->ctx, (unsigned char *)outdata + len, &pad)) {
+		if(EVP_EncryptInit_ex(cipher->ctx, NULL, NULL, NULL, NULL)
+				&& EVP_EncryptUpdate(cipher->ctx, (unsigned char *)outdata, &len, indata, inlen)
+				&& EVP_EncryptFinal(cipher->ctx, (unsigned char *)outdata + len, &pad)) {
 			if(outlen) *outlen = len + pad;
 			return true;
 		}
 	} else {
 		int len;
-		if(EVP_EncryptUpdate(&cipher->ctx, outdata, &len, indata, inlen)) {
+		if(EVP_EncryptUpdate(cipher->ctx, outdata, &len, indata, inlen)) {
 			if(outlen) *outlen = len;
 			return true;
 		}
@@ -140,15 +142,15 @@ bool cipher_encrypt(cipher_t *cipher, const void *indata, size_t inlen, void *ou
 bool cipher_decrypt(cipher_t *cipher, const void *indata, size_t inlen, void *outdata, size_t *outlen, bool oneshot) {
 	if(oneshot) {
 		int len, pad;
-		if(EVP_DecryptInit_ex(&cipher->ctx, NULL, NULL, NULL, NULL)
-				&& EVP_DecryptUpdate(&cipher->ctx, (unsigned char *)outdata, &len, indata, inlen)
-				&& EVP_DecryptFinal(&cipher->ctx, (unsigned char *)outdata + len, &pad)) {
+		if(EVP_DecryptInit_ex(cipher->ctx, NULL, NULL, NULL, NULL)
+				&& EVP_DecryptUpdate(cipher->ctx, (unsigned char *)outdata, &len, indata, inlen)
+				&& EVP_DecryptFinal(cipher->ctx, (unsigned char *)outdata + len, &pad)) {
 			if(outlen) *outlen = len + pad;
 			return true;
 		}
 	} else {
 		int len;
-		if(EVP_EncryptUpdate(&cipher->ctx, outdata, &len, indata, inlen)) {
+		if(EVP_EncryptUpdate(cipher->ctx, outdata, &len, indata, inlen)) {
 			if(outlen) *outlen = len;
 			return true;
 		}
@@ -162,9 +164,9 @@ int cipher_get_nid(const cipher_t *cipher) {
 	if(!cipher || !cipher->cipher)
 		return 0;
 
-	return cipher->cipher->nid;
+	return EVP_CIPHER_nid(cipher->cipher);
 }
 
 bool cipher_active(const cipher_t *cipher) {
-	return cipher && cipher->cipher && cipher->cipher->nid != 0;
+	return cipher && cipher->cipher && EVP_CIPHER_nid(cipher->cipher) != 0;
 }
diff --git a/src/openssl/digest.c b/src/openssl/digest.c
index 0e600b9d..135e7e69 100644
--- a/src/openssl/digest.c
+++ b/src/openssl/digest.c
@@ -93,14 +93,19 @@ bool digest_create(digest_t *digest, const void *indata, size_t inlen, void *out
 			return false;
 		}
 	} else {
-		EVP_MD_CTX ctx;
+		EVP_MD_CTX *ctx = EVP_MD_CTX_create();
+		if(!ctx)
+			abort();
 
-		if(!EVP_DigestInit(&ctx, digest->digest)
-				|| !EVP_DigestUpdate(&ctx, indata, inlen)
-				|| !EVP_DigestFinal(&ctx, tmpdata, NULL)) {
+		if(!EVP_DigestInit(ctx, digest->digest)
+				|| !EVP_DigestUpdate(ctx, indata, inlen)
+				|| !EVP_DigestFinal(ctx, tmpdata, NULL)) {
 			logger(DEBUG_ALWAYS, LOG_DEBUG, "Error creating digest: %s", ERR_error_string(ERR_get_error(), NULL));
+			EVP_MD_CTX_free(ctx);
 			return false;
 		}
+
+		EVP_MD_CTX_free(ctx);
 	}
 
 	memcpy(outdata, tmpdata, digest->maclength);
@@ -118,14 +123,14 @@ int digest_get_nid(const digest_t *digest) {
 	if(!digest || !digest->digest)
 		return 0;
 
-	return digest->digest->type;
+	return EVP_MD_type(digest->digest);
 }
 
 size_t digest_keylength(const digest_t *digest) {
 	if(!digest || !digest->digest)
 		return 0;
 
-	return digest->digest->md_size;
+	return EVP_MD_size(digest->digest);
 }
 
 size_t digest_length(const digest_t *digest) {
@@ -136,5 +141,5 @@ size_t digest_length(const digest_t *digest) {
 }
 
 bool digest_active(const digest_t *digest) {
-	return digest && digest->digest && digest->digest->type != 0;
+	return digest && digest->digest && EVP_MD_type(digest->digest) != 0;
 }
diff --git a/src/openssl/rsa.c b/src/openssl/rsa.c
index 9c1f4984..3033bcd7 100644
--- a/src/openssl/rsa.c
+++ b/src/openssl/rsa.c
@@ -30,28 +30,51 @@ typedef RSA rsa_t;
 
 // Set RSA keys
 
+#ifndef HAVE_RSA_SET0_KEY
+int RSA_set0_key(RSA *r, BIGNUM *n, BIGNUM *e, BIGNUM *d) {
+	BN_free(r->n); r->n = n;
+	BN_free(r->e); r->e = e;
+	BN_free(r->d); r->d = d;
+	return 1;
+}
+#endif
+
 rsa_t *rsa_set_hex_public_key(char *n, char *e) {
-	rsa_t *rsa = RSA_new();
-	if(!rsa)
-		return NULL;
+	BIGNUM *bn_n = NULL;
+	BIGNUM *bn_e = NULL;
 
-	if(BN_hex2bn(&rsa->n, n) != strlen(n) || BN_hex2bn(&rsa->e, e) != strlen(e)) {
-		RSA_free(rsa);
+	if(BN_hex2bn(&bn_n, n) != strlen(n) || BN_hex2bn(&bn_e, e) != strlen(e)) {
+		BN_free(bn_e);
+		BN_free(bn_n);
 		return false;
 	}
 
+	rsa_t *rsa = RSA_new();
+	if(!rsa)
+		return NULL;
+	
+	RSA_set0_key(rsa, bn_n, bn_e, NULL);
+
 	return rsa;
 }
 
 rsa_t *rsa_set_hex_private_key(char *n, char *e, char *d) {
+	BIGNUM *bn_n = NULL;
+	BIGNUM *bn_e = NULL;
+	BIGNUM *bn_d = NULL;
+
+	if(BN_hex2bn(&bn_n, n) != strlen(n) || BN_hex2bn(&bn_e, e) != strlen(e) || BN_hex2bn(&bn_d, d) != strlen(d)) {
+		BN_free(bn_d);
+		BN_free(bn_e);
+		BN_free(bn_n);
+		return false;
+	}
+
 	rsa_t *rsa = RSA_new();
 	if(!rsa)
 		return NULL;
 
-	if(BN_hex2bn(&rsa->n, n) != strlen(n) || BN_hex2bn(&rsa->e, e) != strlen(e) || BN_hex2bn(&rsa->d, d) != strlen(d)) {
-		RSA_free(rsa);
-		return false;
-	}
+	RSA_set0_key(rsa, bn_n, bn_e, bn_d);
 
 	return rsa;
 }
diff --git a/src/openssl/rsagen.c b/src/openssl/rsagen.c
index 3a8c8ad2..4a47619e 100644
--- a/src/openssl/rsagen.c
+++ b/src/openssl/rsagen.c
@@ -30,7 +30,7 @@ typedef RSA rsa_t;
 
 /* This function prettyprints the key generation process */
 
-static void indicator(int a, int b, void *p) {
+static int indicator(int a, int b, BN_GENCB *cb) {
 	switch (a) {
 		case 0:
 			fprintf(stderr, ".");
@@ -62,12 +62,39 @@ static void indicator(int a, int b, void *p) {
 		default:
 			fprintf(stderr, "?");
 	}
+
+	return 1;
 }
 
 // Generate RSA key
 
+#ifndef HAVE_BN_GENCB_NEW
+BN_GENCB *BN_GENCB_new(void) {
+	return xzalloc(sizeof(BN_GENCB));
+}
+
+void BN_GENCB_free(BN_GENCB *cb) {
+	free(cb);
+}
+#endif
+
 rsa_t *rsa_generate(size_t bits, unsigned long exponent) {
-	return RSA_generate_key(bits, exponent, indicator, NULL);
+	BIGNUM *bn_e = BN_new();
+	rsa_t *rsa = RSA_new();
+	BN_GENCB *cb = BN_GENCB_new();
+
+	if(!bn_e || !rsa || !cb)
+		abort();
+
+	BN_set_word(bn_e, exponent);
+	BN_GENCB_set(cb, indicator, NULL);
+
+	RSA_generate_key_ex(rsa, bits, bn_e, cb);
+
+	BN_GENCB_free(cb);
+	BN_free(bn_e);
+
+	return rsa;
 }
 
 // Write PEM RSA keys