Annotation of /trunk/rdesktop/patches/rdesktop-1.8.3-openssl-1.1-fixes.patch
Parent Directory | Revision Log
Revision 3096 -
(hide annotations)
(download)
Tue Mar 27 12:29:19 2018 UTC (6 years, 6 months ago) by niro
File size: 3609 byte(s)
Tue Mar 27 12:29:19 2018 UTC (6 years, 6 months ago) by niro
File size: 3609 byte(s)
-added openssl-1.1 fixes
1 | niro | 3096 | From bd6aa6acddf0ba640a49834807872f4cc0d0a773 Mon Sep 17 00:00:00 2001 |
2 | From: Jani Hakala <jjhakala@gmail.com> | ||
3 | Date: Thu, 16 Jun 2016 14:28:15 +0300 | ||
4 | Subject: [PATCH] Fix OpenSSL 1.1 compability issues | ||
5 | |||
6 | Some data types have been made opaque in OpenSSL version 1.1 so | ||
7 | stack allocation and accessing struct fields directly does not work. | ||
8 | --- | ||
9 | ssl.c | 65 ++++++++++++++++++++++++++++++++++++++++------------------------- | ||
10 | 1 file changed, 40 insertions(+), 25 deletions(-) | ||
11 | |||
12 | diff --git a/ssl.c b/ssl.c | ||
13 | index 4875125..032e9b9 100644 | ||
14 | --- a/ssl.c | ||
15 | +++ b/ssl.c | ||
16 | @@ -88,7 +88,7 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * | ||
17 | uint8 * exponent) | ||
18 | { | ||
19 | BN_CTX *ctx; | ||
20 | - BIGNUM mod, exp, x, y; | ||
21 | + BIGNUM *mod, *exp, *x, *y; | ||
22 | uint8 inr[SEC_MAX_MODULUS_SIZE]; | ||
23 | int outlen; | ||
24 | |||
25 | @@ -98,24 +98,24 @@ rdssl_rsa_encrypt(uint8 * out, uint8 * in, int len, uint32 modulus_size, uint8 * | ||
26 | reverse(inr, len); | ||
27 | |||
28 | ctx = BN_CTX_new(); | ||
29 | - BN_init(&mod); | ||
30 | - BN_init(&exp); | ||
31 | - BN_init(&x); | ||
32 | - BN_init(&y); | ||
33 | - | ||
34 | - BN_bin2bn(modulus, modulus_size, &mod); | ||
35 | - BN_bin2bn(exponent, SEC_EXPONENT_SIZE, &exp); | ||
36 | - BN_bin2bn(inr, len, &x); | ||
37 | - BN_mod_exp(&y, &x, &exp, &mod, ctx); | ||
38 | - outlen = BN_bn2bin(&y, out); | ||
39 | + mod = BN_new(); | ||
40 | + exp = BN_new(); | ||
41 | + x = BN_new(); | ||
42 | + y = BN_new(); | ||
43 | + | ||
44 | + BN_bin2bn(modulus, modulus_size, mod); | ||
45 | + BN_bin2bn(exponent, SEC_EXPONENT_SIZE, exp); | ||
46 | + BN_bin2bn(inr, len, x); | ||
47 | + BN_mod_exp(y, x, exp, mod, ctx); | ||
48 | + outlen = BN_bn2bin(y, out); | ||
49 | reverse(out, outlen); | ||
50 | if (outlen < (int) modulus_size) | ||
51 | memset(out + outlen, 0, modulus_size - outlen); | ||
52 | |||
53 | - BN_free(&y); | ||
54 | - BN_clear_free(&x); | ||
55 | - BN_free(&exp); | ||
56 | - BN_free(&mod); | ||
57 | + BN_free(y); | ||
58 | + BN_clear_free(x); | ||
59 | + BN_free(exp); | ||
60 | + BN_free(mod); | ||
61 | BN_CTX_free(ctx); | ||
62 | } | ||
63 | |||
64 | @@ -146,12 +146,20 @@ rdssl_cert_to_rkey(RDSSL_CERT * cert, uint32 * key_len) | ||
65 | |||
66 | Kudos to Richard Levitte for the following (. intiutive .) | ||
67 | lines of code that resets the OID and let's us extract the key. */ | ||
68 | - nid = OBJ_obj2nid(cert->cert_info->key->algor->algorithm); | ||
69 | + | ||
70 | + X509_PUBKEY *key = NULL; | ||
71 | + X509_ALGOR *algor = NULL; | ||
72 | + | ||
73 | + key = X509_get_X509_PUBKEY(cert); | ||
74 | + algor = X509_PUBKEY_get0_param(NULL, NULL, 0, &algor, key); | ||
75 | + | ||
76 | + nid = OBJ_obj2nid(algor->algorithm); | ||
77 | + | ||
78 | if ((nid == NID_md5WithRSAEncryption) || (nid == NID_shaWithRSAEncryption)) | ||
79 | { | ||
80 | DEBUG_RDP5(("Re-setting algorithm type to RSA in server certificate\n")); | ||
81 | - ASN1_OBJECT_free(cert->cert_info->key->algor->algorithm); | ||
82 | - cert->cert_info->key->algor->algorithm = OBJ_nid2obj(NID_rsaEncryption); | ||
83 | + X509_PUBKEY_set0_param(key, OBJ_nid2obj(NID_rsaEncryption), | ||
84 | + 0, NULL, NULL, 0); | ||
85 | } | ||
86 | epk = X509_get_pubkey(cert); | ||
87 | if (NULL == epk) | ||
88 | @@ -201,14 +209,24 @@ rdssl_rkey_get_exp_mod(RDSSL_RKEY * rkey, uint8 * exponent, uint32 max_exp_len, | ||
89 | { | ||
90 | int len; | ||
91 | |||
92 | - if ((BN_num_bytes(rkey->e) > (int) max_exp_len) || | ||
93 | - (BN_num_bytes(rkey->n) > (int) max_mod_len)) | ||
94 | + BIGNUM *e = NULL; | ||
95 | + BIGNUM *n = NULL; | ||
96 | + | ||
97 | +#if OPENSSL_VERSION_NUMBER < 0x10100000L | ||
98 | + e = rkey->e; | ||
99 | + n = rkey->n; | ||
100 | +#else | ||
101 | + RSA_get0_key(rkey, &e, &n, NULL); | ||
102 | +#endif | ||
103 | + | ||
104 | + if ((BN_num_bytes(e) > (int) max_exp_len) || | ||
105 | + (BN_num_bytes(n) > (int) max_mod_len)) | ||
106 | { | ||
107 | return 1; | ||
108 | } | ||
109 | - len = BN_bn2bin(rkey->e, exponent); | ||
110 | + len = BN_bn2bin(e, exponent); | ||
111 | reverse(exponent, len); | ||
112 | - len = BN_bn2bin(rkey->n, modulus); | ||
113 | + len = BN_bn2bin(n, modulus); | ||
114 | reverse(modulus, len); | ||
115 | return 0; | ||
116 | } | ||
117 | @@ -229,8 +247,5 @@ void | ||
118 | rdssl_hmac_md5(const void *key, int key_len, const unsigned char *msg, int msg_len, | ||
119 | unsigned char *md) | ||
120 | { | ||
121 | - HMAC_CTX ctx; | ||
122 | - HMAC_CTX_init(&ctx); | ||
123 | HMAC(EVP_md5(), key, key_len, msg, msg_len, md, NULL); | ||
124 | - HMAC_CTX_cleanup(&ctx); | ||
125 | } |