Contents of /trunk/rdesktop/patches/rdesktop-1.8.3-openssl-1.1-fixes.patch
Parent Directory | Revision Log
Revision 3096 -
(show annotations)
(download)
Tue Mar 27 12:29:19 2018 UTC (6 years, 6 months ago) by niro
File size: 3609 byte(s)
Tue Mar 27 12:29:19 2018 UTC (6 years, 6 months ago) by niro
File size: 3609 byte(s)
-added openssl-1.1 fixes
1 | From bd6aa6acddf0ba640a49834807872f4cc0d0a773 Mon Sep 17 00:00:00 2001 |
2 | From: Jani Hakala <jjhakala@gmail.com> |
3 | Date: Thu, 16 Jun 2016 14:28:15 +0300 |
4 | Subject: [PATCH] Fix OpenSSL 1.1 compability issues |
5 | |
6 | Some data types have been made opaque in OpenSSL version 1.1 so |
7 | stack allocation and accessing struct fields directly does not work. |
8 | --- |
9 | ssl.c | 65 ++++++++++++++++++++++++++++++++++++++++------------------------- |
10 | 1 file changed, 40 insertions(+), 25 deletions(-) |
11 | |
12 | diff --git a/ssl.c b/ssl.c |
13 | index 4875125..032e9b9 100644 |
14 | --- a/ssl.c |
15 | +++ b/ssl.c |
16 | @@ -88,7 +88,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * |
17 | uint8 * exponent) |
18 | { |
19 | BN_CTX *ctx; |
20 | - BIGNUM mod, exp, x, y; |
21 | + BIGNUM *mod, *exp, *x, *y; |
22 | uint8 inr[SEC_MAX_MODULUS_SIZE]; |
23 | int outlen; |
24 | |
25 | @@ -98,24 +98,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * |
26 | reverse(inr, len); |
27 | |
28 | ctx = BN_CTX_new(); |
29 | - BN_init(&mod); |
30 | - BN_init(&exp); |
31 | - BN_init(&x); |
32 | - BN_init(&y); |
33 | - |
34 | - BN_bin2bn(modulus, modulus_size, &mod); |
35 | - BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp); |
36 | - BN_bin2bn(inr, len, &x); |
37 | - BN_mod_exp(&y, &x, &exp, &mod, ctx); |
38 | - outlen = BN_bn2bin(&y, out); |
39 | + mod = BN_new(); |
40 | + exp = BN_new(); |
41 | + x = BN_new(); |
42 | + y = BN_new(); |
43 | + |
44 | + BN_bin2bn(modulus, modulus_size, mod); |
45 | + BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp); |
46 | + BN_bin2bn(inr, len, x); |
47 | + BN_mod_exp(y, x, exp, mod, ctx); |
48 | + outlen = BN_bn2bin(y, out); |
49 | reverse(out, outlen); |
50 | if (outlen < (int) modulus_size) |
51 | memset(out + outlen, 0, modulus_size - outlen); |
52 | |
53 | - BN_free(&y); |
54 | - BN_clear_free(&x); |
55 | - BN_free(&exp); |
56 | - BN_free(&mod); |
57 | + BN_free(y); |
58 | + BN_clear_free(x); |
59 | + BN_free(exp); |
60 | + BN_free(mod); |
61 | BN_CTX_free(ctx); |
62 | } |
63 | |
64 | @@ -146,12 +146,20 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len) |
65 | |
66 | Kudos to Richard Levitte for the following (. intiutive .) |
67 | lines of code that resets the OID and let's us extract the key. */ |
68 | - nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm); |
69 | + |
70 | + X509_PUBKEY *key = NULL; |
71 | + X509_ALGOR *algor = NULL; |
72 | + |
73 | + key = X509_get_X509_PUBKEY(cert); |
74 | + algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key); |
75 | + |
76 | + nid = OBJ_obj2nid(algor->algorithm); |
77 | + |
78 | if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption)) |
79 | { |
80 | DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n")); |
81 | - ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm); |
82 | - cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption); |
83 | + X509_PUBKEY_set0_param(key, OBJ_nid2obj(NID_rsaEncryption), |
84 | + 0, NULL, NULL, 0); |
85 | } |
86 | epk = X509_get_pubkey(cert); |
87 | if (NULL == epk) |
88 | @@ -201,14 +209,24 @@ rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, |
89 | { |
90 | int len; |
91 | |
92 | - if ((BN_num_bytes(rkey->e) > (int) max_exp_len) || |
93 | - (BN_num_bytes(rkey->n) > (int) max_mod_len)) |
94 | + BIGNUM *e = NULL; |
95 | + BIGNUM *n = NULL; |
96 | + |
97 | +#if OPENSSL_VERSION_NUMBER < 0x10100000L |
98 | + e = rkey->e; |
99 | + n = rkey->n; |
100 | +#else |
101 | + RSA_get0_key(rkey, &e, &n, NULL); |
102 | +#endif |
103 | + |
104 | + if ((BN_num_bytes(e) > (int) max_exp_len) || |
105 | + (BN_num_bytes(n) > (int) max_mod_len)) |
106 | { |
107 | return 1; |
108 | } |
109 | - len = BN_bn2bin(rkey->e, exponent); |
110 | + len = BN_bn2bin(e, exponent); |
111 | reverse(exponent, len); |
112 | - len = BN_bn2bin(rkey->n, modulus); |
113 | + len = BN_bn2bin(n, modulus); |
114 | reverse(modulus, len); |
115 | return 0; |
116 | } |
117 | @@ -229,8 +247,5 @@ void |
118 | rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len, |
119 | unsigned char *md) |
120 | { |
121 | - HMAC_CTX ctx; |
122 | - HMAC_CTX_init(&ctx); |
123 | HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL); |
124 | - HMAC_CTX_cleanup(&ctx); |
125 | } |