Fix compilation with OpenSSL 1.1.0-dev.

OpenSSL changed the API:
 * 5998e29035
 * b0700d2c8d

changes/bug17549

@@ -0,0 +1,3 @@
+  o Minor bugfixes (compilation):
+    - Repair compilation with the most recent (unreleased, alpha)
+      vesions of OpenSSL 1.1. Fixes bug 17549.

src/common/crypto.c

@@ -227,7 +227,11 @@ const char *
 crypto_openssl_get_version_str(void)
 {
   if (crypto_openssl_version_str == NULL) {
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
+    const char *raw_version = OpenSSL_version(OPENSSL_VERSION);
+#else
     const char *raw_version = SSLeay_version(SSLEAY_VERSION);
+#endif
     crypto_openssl_version_str = parse_openssl_version_str(raw_version);
   }
   return crypto_openssl_version_str;
@@ -251,11 +255,17 @@ crypto_openssl_get_header_version_str(void)
 static int
 crypto_force_rand_ssleay(void)
 {
-  if (RAND_get_rand_method() != RAND_SSLeay()) {
+  RAND_METHOD *default_method;
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
+  default_method = RAND_OpenSSL();
+#else
+  default_method = RAND_SSLeay();
+#endif
+  if (RAND_get_rand_method() != default_method) {
     log_notice(LD_CRYPTO, "It appears that one of our engines has provided "
                "a replacement the OpenSSL RNG. Resetting it to the default "
                "implementation.");
-    RAND_set_rand_method(RAND_SSLeay());
+    RAND_set_rand_method(default_method);
     return 1;
   }
   return 0;
@@ -291,16 +301,23 @@ crypto_early_init(void)
 
     setup_openssl_threading();
 
-    if (SSLeay() == OPENSSL_VERSION_NUMBER &&
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
-        !strcmp(SSLeay_version(SSLEAY_VERSION), OPENSSL_VERSION_TEXT)) {
+    unsigned long version_num = OpenSSL_version_num();
+    const char *version_str = OpenSSL_version(OPENSSL_VERSION);
+#else
+    unsigned long version_num = SSLeay();
+    const char *version_str = SSLeay_version(SSLEAY_VERSION);
+#endif
+    if (version_num == OPENSSL_VERSION_NUMBER &&
+        !strcmp(version_str, OPENSSL_VERSION_TEXT)) {
       log_info(LD_CRYPTO, "OpenSSL version matches version from headers "
-                 "(%lx: %s).", SSLeay(), SSLeay_version(SSLEAY_VERSION));
+                 "(%lx: %s).", version_num, version_str);
     } else {
       log_warn(LD_CRYPTO, "OpenSSL version from headers does not match the "
                "version we're running with. If you get weird crashes, that "
                "might be why. (Compiled with %lx: %s; running with %lx: %s).",
                (unsigned long)OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_TEXT,
-               SSLeay(), SSLeay_version(SSLEAY_VERSION));
+               version_num, version_str);
     }
 
     crypto_force_rand_ssleay();

src/common/tortls.c

@@ -384,7 +384,11 @@ tor_tls_init(void)
 
 #if (SIZEOF_VOID_P >= 8 &&                              \
      OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,0,1))
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
+    long version = OpenSSL_version_num();
+#else
     long version = SSLeay();
+#endif
 
     /* LCOV_EXCL_START : we can't test these lines on the same machine */
     if (version >= OPENSSL_V_SERIES(1,0,1)) {
@@ -1525,7 +1529,6 @@ STATIC void
 tor_tls_server_info_callback(const SSL *ssl, int type, int val)
 {
   tor_tls_t *tls;
-  int ssl_state;
   (void) val;
 
   tor_tls_debug_state_callback(ssl, type, val);
@@ -1533,10 +1536,16 @@ tor_tls_server_info_callback(const SSL *ssl, int type, int val)
   if (type != SSL_CB_ACCEPT_LOOP)
     return;
 
-  ssl_state = SSL_state(ssl);
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
+  OSSL_HANDSHAKE_STATE ssl_state = SSL_get_state(ssl);
+  if (ssl_state == TLS_ST_SW_SRVR_HELLO)
+    return;
+#else
+  int ssl_state = SSL_state(ssl);
   if ((ssl_state != SSL3_ST_SW_SRVR_HELLO_A) &&
       (ssl_state != SSL3_ST_SW_SRVR_HELLO_B))
     return;
+#endif
   tls = tor_tls_get_by_ssl(ssl);
   if (tls) {
     /* Check whether we're watching for renegotiates.  If so, this is one! */
@@ -1892,13 +1901,16 @@ int
 tor_tls_handshake(tor_tls_t *tls)
 {
   int r;
-  int oldstate;
   tor_assert(tls);
   tor_assert(tls->ssl);
   tor_assert(tls->state == TOR_TLS_ST_HANDSHAKE);
 
   check_no_tls_errors();
-  oldstate = SSL_state(tls->ssl);
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
+  OSSL_HANDSHAKE_STATE oldstate = SSL_get_state(tls->ssl);
+#else
+  int oldstate = SSL_state(tls->ssl);
+#endif
   if (tls->isServer) {
     log_debug(LD_HANDSHAKE, "About to call SSL_accept on %p (%s)", tls,
               SSL_state_string_long(tls->ssl));
@@ -1908,7 +1920,12 @@ tor_tls_handshake(tor_tls_t *tls)
               SSL_state_string_long(tls->ssl));
     r = SSL_connect(tls->ssl);
   }
-  if (oldstate != SSL_state(tls->ssl))
+#if OPENSSL_VERSION_NUMBER >= OPENSSL_V_SERIES(1,1,0)
+  OSSL_HANDSHAKE_STATE newstate = SSL_get_state(tls->ssl);
+#else
+  int newstate = SSL_state(tls->ssl);
+#endif
+  if (oldstate != newstate)
     log_debug(LD_HANDSHAKE, "After call, %p was in state %s",
               tls, SSL_state_string_long(tls->ssl));
   /* We need to call this here and not earlier, since OpenSSL has a penchant