Merge remote-tracking branch 'tor-github/pr/508'

.gitignore

@@ -155,6 +155,8 @@ uptime-*.json
 
 # /src/lib
 /src/lib/libcurve25519_donna.a
+/src/lib/libtor-buf.a
+/src/lib/libtor-buf-testing.a
 /src/lib/libtor-compress.a
 /src/lib/libtor-compress-testing.a
 /src/lib/libtor-container.a

Makefile.am

@@ -40,6 +40,7 @@ endif
 TOR_UTIL_LIBS = \
         src/lib/libtor-geoip.a \
 	src/lib/libtor-process.a \
+        src/lib/libtor-buf.a \
 	src/lib/libtor-time.a \
 	src/lib/libtor-fs.a \
 	src/lib/libtor-encoding.a \
@@ -70,6 +71,7 @@ if UNITTESTS_ENABLED
 TOR_UTIL_TESTING_LIBS = \
         src/lib/libtor-geoip-testing.a \
 	src/lib/libtor-process-testing.a \
+        src/lib/libtor-buf-testing.a \
 	src/lib/libtor-time-testing.a \
 	src/lib/libtor-fs-testing.a \
 	src/lib/libtor-encoding-testing.a \

changes/ticket28362

@@ -0,0 +1,6 @@
+  o Code simplification and refactoring:
+    - The .may_include files that we use to describe our
+      directory-by-directory dependency structure now describe a noncircular
+      dependency graph over the directories that they cover.
+      Our checkIncludes.py tool now enforces this.
+      Closes ticket 28362.

scripts/maint/checkIncludes.py

@@ -33,6 +33,9 @@ else:
     def open_file(fname):
         return open(fname, 'r', encoding='utf-8')
 
+def warn(msg):
+    print(msg, file=sys.stderr)
+
 def err(msg):
     """ Declare that an error has happened, and remember that there has
         been an error. """
@@ -48,14 +51,34 @@ def fname_is_c(fname):
 INCLUDE_PATTERN = re.compile(r'\s*#\s*include\s+"([^"]*)"')
 RULES_FNAME = ".may_include"
 
+ALLOWED_PATTERNS = [
+    re.compile(r'^.*\*\.(h|inc)$'),
+    re.compile(r'^.*/.*\.h$'),
+    re.compile(r'^ext/.*\.c$'),
+    re.compile(r'^orconfig.h$'),
+    re.compile(r'^micro-revision.i$'),
+]
+
+def pattern_is_normal(s):
+    for p in ALLOWED_PATTERNS:
+        if p.match(s):
+            return True
+    return False
+
 class Rules(object):
     """ A 'Rules' object is the parsed version of a .may_include file. """
     def __init__(self, dirpath):
         self.dirpath = dirpath
+        if dirpath.startswith("src/"):
+            self.incpath = dirpath[4:]
+        else:
+            self.incpath = dirpath
         self.patterns = []
         self.usedPatterns = set()
 
     def addPattern(self, pattern):
+        if not pattern_is_normal(pattern):
+            warn("Unusual pattern {} in {}".format(pattern, self.dirpath))
         self.patterns.append(pattern)
 
     def includeOk(self, path):
@@ -86,6 +109,20 @@ class Rules(object):
             if p not in self.usedPatterns:
                 print("Pattern {} in {} was never used.".format(p, self.dirpath))
 
+    def getAllowedDirectories(self):
+        allowed = []
+        for p in self.patterns:
+            m = re.match(r'^(.*)/\*\.(h|inc)$', p)
+            if m:
+                allowed.append(m.group(1))
+                continue
+            m = re.match(r'^(.*)/[^/]*$', p)
+            if m:
+                allowed.append(m.group(1))
+                continue
+
+        return allowed
+
 def load_include_rules(fname):
     """ Read a rules file from 'fname', and return it as a Rules object. """
     result = Rules(os.path.split(fname)[0])
@@ -98,6 +135,9 @@ def load_include_rules(fname):
     return result
 
 list_unused = False
+log_sorted_levels = False
+
+uses_dirs = { }
 
 for dirpath, dirnames, fnames in os.walk("src"):
     if ".may_include" in fnames:
@@ -108,8 +148,34 @@ for dirpath, dirnames, fnames in os.walk("src"):
         if list_unused:
             rules.noteUnusedRules()
 
+        uses_dirs[rules.incpath] = rules.getAllowedDirectories()
+
 if trouble:
     err(
 """To change which includes are allowed in a C file, edit the {}
 files in its enclosing directory.""".format(RULES_FNAME))
     sys.exit(1)
+
+all_levels = []
+
+n = 0
+while uses_dirs:
+    n += 0
+    cur_level = []
+    for k in list(uses_dirs):
+        uses_dirs[k] = [ d for d in uses_dirs[k]
+                         if (d in uses_dirs and d != k)]
+        if uses_dirs[k] == []:
+            cur_level.append(k)
+    for k in cur_level:
+        del uses_dirs[k]
+    n += 1
+    if cur_level and log_sorted_levels:
+        print(n, cur_level)
+    if n > 100:
+        break
+
+if uses_dirs:
+    print("There are circular .may_include dependencies in here somewhere:",
+          uses_dirs)
+    sys.exit(1)

src/app/main/main.c

@@ -67,10 +67,11 @@
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
 #include "lib/compress/compress.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_s2k.h"
 #include "lib/geoip/geoip.h"
+#include "lib/net/resolve.h"
 
 #include "lib/process/waitpid.h"
 
@@ -1417,6 +1418,7 @@ tor_run_main(const tor_main_configuration_t *tor_cfg)
       tor_free_all(0);
       return -1;
     }
+    tor_make_getaddrinfo_cache_active();
 
     // registering libevent rng
 #ifdef HAVE_EVUTIL_SECURE_RNG_SET_URANDOM_DEVICE_FILE

src/core/mainloop/connection.c

@@ -57,7 +57,7 @@
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/err/backtrace.h"
 

src/core/mainloop/mainloop.c

@@ -95,7 +95,7 @@
 #include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/err/backtrace.h"
 #include "lib/tls/buffers_tls.h"

src/core/or/circuitlist.c

@@ -94,7 +94,7 @@
 #include "lib/compress/compress_lzma.h"
 #include "lib/compress/compress_zlib.h"
 #include "lib/compress/compress_zstd.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "ht.h"
 

src/core/or/connection_edge.c

@@ -98,7 +98,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/predict_ports.h"
 #include "feature/stats/rephist.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_util.h"
 
 #include "core/or/cell_st.h"

src/core/or/connection_or.c

@@ -22,7 +22,7 @@
  **/
 #include "core/or/or.h"
 #include "feature/client/bridges.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 /*
  * Define this so we get channel internal functions, since we're implementing
  * part of a subclass (channel_tls_t).

src/core/or/or.h

@@ -26,7 +26,7 @@
 #include "lib/cc/compat_compiler.h"
 #include "lib/cc/torint.h"
 #include "lib/container/map.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/container/smartlist.h"
 #include "lib/crypt_ops/crypto_cipher.h"
 #include "lib/crypt_ops/crypto_rsa.h"

src/core/or/relay.c

@@ -49,7 +49,7 @@
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "feature/client/circpathbias.h"
 #include "core/or/circuitbuild.h"

src/core/or/scheduler.c

@@ -9,7 +9,7 @@
 #define SCHEDULER_KIST_PRIVATE
 #include "core/or/scheduler.h"
 #include "core/mainloop/mainloop.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #define TOR_CHANNEL_INTERNAL_
 #include "core/or/channeltls.h"
 #include "lib/evloop/compat_libevent.h"

src/core/or/scheduler_kist.c

@@ -4,7 +4,7 @@
 #define SCHEDULER_KIST_PRIVATE
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/nodelist/networkstatus.h"

src/core/proto/proto_cell.c

@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_cell.h"
 
 #include "core/or/connection_or.h"

src/core/proto/proto_control0.c

@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_control0.h"
 
 /** Return 1 iff buf looks more like it has an (obsolete) v0 controller

src/core/proto/proto_ext_or.c

@@ -5,7 +5,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_ext_or.h"
 

src/core/proto/proto_http.c

@@ -6,7 +6,7 @@
 
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 
 /** Return true if <b>cmd</b> looks like a HTTP (proxy) request. */

src/core/proto/proto_socks.c

@@ -6,7 +6,7 @@
 
 #include "core/or/or.h"
 #include "feature/client/addressmap.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
 #include "feature/control/control.h"
 #include "app/config/config.h"

src/ext/.may_include

@@ -0,0 +1,10 @@
+
+orconfig.h
+
+lib/err/*.h
+lib/cc/*.h
+
+tinytest*.h
+ext/siphash.h
+ext/byteorder.h
+ext/tor_readpassphrase.h

src/ext/csiphash.c

@@ -30,12 +30,12 @@
 */
 
 #include "lib/cc/torint.h"
-#include "lib/log/util_bug.h"
+#include "lib/err/torerr.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 #include <string.h>
 #include <stdlib.h>
-#include "byteorder.h"
+#include "ext/byteorder.h"
 
 #define ROTATE(x, b) (uint64_t)( ((x) << (b)) | ( (x) >> (64 - (b))) )
 
@@ -112,13 +112,13 @@ static int the_siphash_key_is_set = 0;
 static struct sipkey the_siphash_key;
 
 uint64_t siphash24g(const void *src, unsigned long src_sz) {
-	tor_assert(the_siphash_key_is_set);
+	raw_assert(the_siphash_key_is_set);
 	return siphash24(src, src_sz, &the_siphash_key);
 }
 
 void siphash_set_global_key(const struct sipkey *key)
 {
-	tor_assert(! the_siphash_key_is_set);
+	raw_assert(! the_siphash_key_is_set);
 	the_siphash_key.k0 = key->k0;
 	the_siphash_key.k1 = key->k1;
 	the_siphash_key_is_set = 1;

src/ext/readpassphrase.c

@@ -30,7 +30,7 @@
 #include <signal.h>
 #include <ctype.h>
 #include <fcntl.h>
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #include <errno.h>
 #include <string.h>
 #include <unistd.h>

src/ext/timeouts/.may_include

@@ -0,0 +1,6 @@
+orconfig.h
+
+ext/tor_queue.h
+timeout-bitops.c
+timeout-debug.h
+timeout.h

src/ext/timeouts/timeout.c

@@ -38,7 +38,7 @@
 
 #include <errno.h>     /* errno */
 
-#include "tor_queue.h" /* TAILQ(3) */
+#include "ext/tor_queue.h" /* TAILQ(3) */
 
 #include "timeout.h"
 
@@ -531,7 +531,7 @@ static timeout_t timeouts_int(struct timeouts *T) {
 			timeout = MIN(_timeout, timeout);
 		}
 
-		relmask <<= WHEEL_BIT; 
+		relmask <<= WHEEL_BIT;
 		relmask |= WHEEL_MASK;
 	}
 
@@ -751,4 +751,3 @@ TIMEOUT_PUBLIC int timeout_v_abi(void) {
 TIMEOUT_PUBLIC int timeout_v_api(void) {
 	return TIMEOUT_V_API;
 } /* timeout_version() */
-

src/ext/timeouts/timeout.h

@@ -31,7 +31,7 @@
 
 #include <inttypes.h>   /* PRIu64 PRIx64 PRIX64 uint64_t */
 
-#include "tor_queue.h"  /* TAILQ(3) */
+#include "ext/tor_queue.h"  /* TAILQ(3) */
 
 
 /*
@@ -147,7 +147,7 @@ TIMEOUT_PUBLIC struct timeout *timeout_init(struct timeout *, int);
 #ifndef TIMEOUT_DISABLE_RELATIVE_ACCESS
 TIMEOUT_PUBLIC bool timeout_pending(struct timeout *);
 /* true if on timing wheel, false otherwise */
- 
+
 TIMEOUT_PUBLIC bool timeout_expired(struct timeout *);
 /* true if on expired queue, false otherwise */
 

src/feature/control/control.c

@@ -87,7 +87,7 @@
 #include "feature/rend/rendservice.h"
 #include "feature/stats/geoip_stats.h"
 #include "feature/stats/predict_ports.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/crypt_ops/crypto_util.h"
 #include "lib/encoding/confline.h"

src/feature/stats/geoip_stats.c

@@ -30,7 +30,7 @@
 #include "core/or/or.h"
 
 #include "ht.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "feature/control/control.h"
 #include "feature/client/dnsserv.h"

src/include.am

@@ -1,5 +1,6 @@
 include src/ext/include.am
 include src/lib/arch/include.am
+include src/lib/buf/include.am
 include src/lib/err/include.am
 include src/lib/cc/include.am
 include src/lib/ctime/include.am

src/lib/buf/.may_include

@@ -0,0 +1,10 @@
+orconfig.h
+
+lib/buf/*.h
+lib/cc/*.h
+lib/ctime/*.h
+lib/malloc/*.h
+lib/testsupport/*.h
+lib/log/*.h
+lib/string/*.h
+lib/time/*.h

src/lib/buf/buffers.c

@@ -25,7 +25,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"

src/lib/buf/include.am

@@ -0,0 +1,17 @@
+
+noinst_LIBRARIES += src/lib/libtor-buf.a
+
+if UNITTESTS_ENABLED
+noinst_LIBRARIES += src/lib/libtor-buf-testing.a
+endif
+
+src_lib_libtor_buf_a_SOURCES =			\
+	src/lib/buf/buffers.c
+
+src_lib_libtor_buf_testing_a_SOURCES = \
+	$(src_lib_libtor_buf_a_SOURCES)
+src_lib_libtor_buf_testing_a_CPPFLAGS = $(AM_CPPFLAGS) $(TEST_CPPFLAGS)
+src_lib_libtor_buf_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
+
+noinst_HEADERS +=				\
+	src/lib/buf/buffers.h

src/lib/compress/.may_include

@@ -1,5 +1,6 @@
 orconfig.h
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/compress/*.h
 lib/container/*.h

src/lib/compress/compress_buf.c

@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/cc/compat_compiler.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/compress/compress.h"
 #include "lib/log/util_bug.h"
 

src/lib/container/.may_include

@@ -7,12 +7,9 @@ lib/malloc/*.h
 lib/err/*.h
 lib/smartlist_core/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
 lib/intmath/*.h
 lib/log/*.h
 
-# XXXX I am unsure about this one. It's only here for buffers.c
-lib/time/*.h
-
-ht.h
-siphash.h
+ext/ht.h
+ext/siphash.h

src/lib/container/bloomfilt.c

@@ -14,7 +14,7 @@
 #include "lib/container/bloomfilt.h"
 #include "lib/intmath/bits.h"
 #include "lib/log/util_bug.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** How many bloom-filter bits we set per address. This is twice the
  * BLOOMFILT_N_HASHES value, since we split the siphash output into two 32-bit

src/lib/container/include.am

@@ -7,7 +7,6 @@ endif
 
 src_lib_libtor_container_a_SOURCES =			\
 	src/lib/container/bloomfilt.c			\
-	src/lib/container/buffers.c			\
 	src/lib/container/map.c				\
 	src/lib/container/order.c			\
 	src/lib/container/smartlist.c
@@ -20,7 +19,6 @@ src_lib_libtor_container_testing_a_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 noinst_HEADERS +=					\
 	src/lib/container/bitarray.h			\
 	src/lib/container/bloomfilt.h			\
-	src/lib/container/buffers.h			\
 	src/lib/container/handles.h			\
 	src/lib/container/map.h				\
 	src/lib/container/order.h			\

src/lib/container/map.c

@@ -21,7 +21,7 @@
 #include <stdlib.h>
 #include <string.h>
 
-#include "ht.h"
+#include "ext/ht.h"
 
 /** Helper: Declare an entry type and a map type to implement a mapping using
  * ht.h.  The map type will be called <b>maptype</b>.  The key part of each

src/lib/container/map.h

@@ -15,7 +15,7 @@
 #include "lib/testsupport/testsupport.h"
 #include "lib/cc/torint.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #define DECLARE_MAP_FNS(maptype, keytype, prefix)                       \
   typedef struct maptype maptype;                                       \

src/lib/crypt_ops/.may_include

@@ -13,7 +13,7 @@ lib/intmath/*.h
 lib/sandbox/*.h
 lib/string/*.h
 lib/subsys/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
 lib/thread/*.h
 lib/log/*.h
 
@@ -22,4 +22,4 @@ trunnel/pwbox.h
 keccak-tiny/*.h
 ed25519/*.h
 
-siphash.h
+ext/siphash.h

src/lib/crypt_ops/crypto_init.c

@@ -24,7 +24,7 @@
 
 #include "lib/subsys/subsys.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /** Boolean: has our crypto library been initialized? (early phase) */
 static int crypto_early_initialized_ = 0;

src/lib/crypt_ops/digestset.c

@@ -11,7 +11,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/defs/digest_sizes.h"
 #include "lib/crypt_ops/digestset.h"
-#include "siphash.h"
+#include "ext/siphash.h"
 
 /* Wrap our hash function to have the signature that the bloom filter
  * needs. */

src/lib/evloop/.may_include

@@ -12,5 +12,5 @@ lib/testsupport/*.h
 lib/thread/*.h
 lib/time/*.h
 
-src/ext/timeouts/timeout.c
-tor_queue.h
+ext/timeouts/timeout.c
+ext/tor_queue.h

src/lib/evloop/timers.c

@@ -80,7 +80,8 @@ struct timeout_cb {
  * use 32-bit math. */
 #define WHEEL_BIT 5
 #endif
-#include "src/ext/timeouts/timeout.c"
+
+#include "ext/timeouts/timeout.c"
 
 static struct timeouts *global_timeouts = NULL;
 static struct mainloop_event_t *global_timer_event = NULL;

src/lib/evloop/workqueue.c

@@ -36,7 +36,7 @@
 #include "lib/net/socket.h"
 #include "lib/thread/threads.h"
 
-#include "tor_queue.h"
+#include "ext/tor_queue.h"
 #include <event2/event.h>
 #include <string.h>
 

src/lib/fs/.may_include

@@ -13,4 +13,4 @@ lib/malloc/*.h
 lib/memarea/*.h
 lib/sandbox/*.h
 lib/string/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h

src/lib/malloc/.may_include

@@ -3,4 +3,4 @@ orconfig.h
 lib/cc/*.h
 lib/err/*.h
 lib/malloc/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h

src/lib/memarea/.may_include

@@ -1,7 +1,7 @@
 orconfig.h
 lib/arch/*.h
 lib/cc/*.h
-lib/container/*.h
 lib/log/*.h
 lib/malloc/*.h
 lib/memarea/*.h
+lib/smartlist_core/*.h

src/lib/memarea/memarea.c

@@ -16,7 +16,8 @@
 
 #include "lib/arch/bytes.h"
 #include "lib/cc/torint.h"
-#include "lib/container/smartlist.h"
+#include "lib/smartlist_core/smartlist_core.h"
+#include "lib/smartlist_core/smartlist_foreach.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"

src/lib/net/.may_include

@@ -1,8 +1,9 @@
 orconfig.h
-siphash.h
-ht.h
+ext/siphash.h
+ext/ht.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/ctime/*.h

src/lib/net/address.c

@@ -53,7 +53,7 @@
 #include "lib/string/printf.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
+#include "ext/siphash.h"
 
 #ifdef HAVE_SYS_TIME_H
 #include <sys/time.h>

src/lib/net/buffers_net.c

@@ -11,7 +11,7 @@
 
 #define BUFFERS_PRIVATE
 #include "lib/net/buffers_net.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/net/nettypes.h"

src/lib/net/resolve.c

@@ -16,8 +16,8 @@
 #include "lib/string/parse_int.h"
 #include "lib/string/util_string.h"
 
-#include "siphash.h"
-#include "ht.h"
+#include "ext/siphash.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_TYPES_H
 #include <sys/types.h>
@@ -421,4 +421,13 @@ tor_make_getaddrinfo_cache_active(void)
 {
   sandbox_getaddrinfo_is_active = 1;
 }
+#else
+void
+sandbox_disable_getaddrinfo_cache(void)
+{
+}
+void
+tor_make_getaddrinfo_cache_active(void)
+{
+}
 #endif

src/lib/net/resolve.h

@@ -42,7 +42,6 @@ int tor_getaddrinfo(const char *name, const char *servname,
                         struct addrinfo **res);
 void tor_freeaddrinfo(struct addrinfo *addrinfo);
 void tor_free_getaddrinfo_cache(void);
-void tor_make_getaddrinfo_cache_active(void);
 #else /* !(defined(USE_SANDBOX_GETADDRINFO)) */
 #define tor_getaddrinfo(name, servname, hints, res)  \
   getaddrinfo((name),(servname), (hints),(res))
@@ -54,5 +53,6 @@ void tor_make_getaddrinfo_cache_active(void);
 #endif /* defined(USE_SANDBOX_GETADDRINFO) */
 
 void sandbox_disable_getaddrinfo_cache(void);
+void tor_make_getaddrinfo_cache_active(void);
 
 #endif

src/lib/process/.may_include

@@ -15,4 +15,4 @@ lib/subsys/*.h
 lib/testsupport/*.h
 lib/thread/*.h
 
-ht.h
+ext/ht.h

src/lib/process/waitpid.c

@@ -16,7 +16,7 @@
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
 #include "lib/malloc/malloc.h"
-#include "ht.h"
+#include "ext/ht.h"
 
 #ifdef HAVE_SYS_WAIT_H
 #include <sys/wait.h>

src/lib/sandbox/.may_include

@@ -5,11 +5,10 @@ lib/container/*.h
 lib/err/*.h
 lib/log/*.h
 lib/malloc/*.h
-lib/net/*.h
 lib/sandbox/*.h
 lib/sandbox/*.inc
 lib/string/*.h
 
-ht.h
-siphash.h
-tor_queue.h
+ext/ht.h
+ext/siphash.h
+ext/tor_queue.h

src/lib/sandbox/sandbox.c

@@ -38,13 +38,12 @@
 #include "lib/err/torerr.h"
 #include "lib/log/log.h"
 #include "lib/cc/torint.h"
-#include "lib/net/resolve.h"
 #include "lib/malloc/malloc.h"
 #include "lib/string/scanf.h"
 
-#include "tor_queue.h"
-#include "ht.h"
-#include "siphash.h"
+#include "ext/tor_queue.h"
+#include "ext/ht.h"
+#include "ext/siphash.h"
 
 #define DEBUGGING_CLOSE
 
@@ -1553,7 +1552,6 @@ install_syscall_filter(sandbox_cfg_t* cfg)
 
   // marking the sandbox as active
   sandbox_active = 1;
-  tor_make_getaddrinfo_cache_active();
 
  end:
   seccomp_release(ctx);
@@ -1800,9 +1798,4 @@ sandbox_is_active(void)
   return 0;
 }
 
-void
-sandbox_disable_getaddrinfo_cache(void)
-{
-}
-
 #endif /* !defined(USE_LIBSECCOMP) */

src/lib/smartlist_core/.may_include

@@ -4,4 +4,4 @@ lib/malloc/*.h
 lib/err/*.h
 lib/string/*.h
 lib/smartlist_core/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h

src/lib/string/.may_include

@@ -6,5 +6,5 @@ lib/malloc/*.h
 lib/ctime/*.h
 lib/string/*.h
 
-strlcat.c
-strlcpy.c
+ext/strlcat.c
+ext/strlcpy.c

src/lib/string/compat_string.c

@@ -14,10 +14,10 @@
 
 /* Inline the strl functions if the platform doesn't have them. */
 #ifndef HAVE_STRLCPY
-#include "strlcpy.c"
+#include "ext/strlcpy.c"
 #endif
 #ifndef HAVE_STRLCAT
-#include "strlcat.c"
+#include "ext/strlcat.c"
 #endif
 
 #include <stdlib.h>

src/lib/term/.may_include

@@ -5,5 +5,4 @@ lib/log/*.h
 lib/term/*.h
 lib/malloc/*.h
 
-# From src/ext
-tor_readpassphrase.h
+ext/tor_readpassphrase.h

src/lib/term/getpass.c

@@ -36,7 +36,7 @@ SecureZeroMemory(PVOID ptr, SIZE_T cnt)
 #elif defined(HAVE_READPASSPHRASE_H)
 #include <readpassphrase.h>
 #else
-#include "tor_readpassphrase.h"
+#include "ext/tor_readpassphrase.h"
 #endif /* defined(_WIN32) || ... */
 
 #include <stdlib.h>

src/lib/tls/.may_include

@@ -1,6 +1,7 @@
 orconfig.h
 
 lib/arch/*.h
+lib/buf/*.h
 lib/cc/*.h
 lib/container/*.h
 lib/crypt_ops/*.h
@@ -12,7 +13,6 @@ lib/malloc/*.h
 lib/net/*.h
 lib/string/*.h
 lib/subsys/*.h
-lib/testsupport/testsupport.h
+lib/testsupport/*.h
 lib/tls/*.h
-
-ciphers.inc
+lib/tls/*.inc

src/lib/tls/buffers_tls.c

@@ -12,7 +12,7 @@
 #define BUFFERS_PRIVATE
 #include "orconfig.h"
 #include <stddef.h>
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/cc/torint.h"
 #include "lib/log/log.h"

src/lib/tls/tortls_openssl.c

@@ -461,7 +461,7 @@ static const char UNRESTRICTED_SERVER_CIPHER_LIST[] =
 /** List of ciphers that clients should advertise, omitting items that
  * our OpenSSL doesn't know about. */
 static const char CLIENT_CIPHER_LIST[] =
-#include "ciphers.inc"
+#include "lib/tls/ciphers.inc"
   /* Tell it not to use SSLv2 ciphers, so that it can select an SSLv3 version
    * of any cipher we say. */
   "!SSLv2"

src/rust/build.rs

@@ -149,8 +149,9 @@ pub fn main() {
             cfg.component("tor-sandbox-testing");
             cfg.component("tor-encoding-testing");
             cfg.component("tor-fs-testing");
-            cfg.component("tor-time-testing");
             cfg.component("tor-net-testing");
+            cfg.component("tor-buf-testing");
+            cfg.component("tor-time-testing");
             cfg.component("tor-thread-testing");
             cfg.component("tor-memarea-testing");
             cfg.component("tor-log-testing");

src/test/fuzz/fuzz_http.c

@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "feature/dircache/dircache.h"

src/test/fuzz/fuzz_http_connect.c

@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_edge.h"

src/test/fuzz/fuzz_socks.c

@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/err/backtrace.h"
 #include "lib/log/log.h"
 #include "core/proto/proto_socks.h"

src/test/test.c

@@ -37,7 +37,7 @@
 
 #include "core/or/or.h"
 #include "lib/err/backtrace.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "core/or/circuitstats.h"
 #include "lib/compress/compress.h"

src/test/test_buffers.c

@@ -6,7 +6,7 @@
 #define BUFFERS_PRIVATE
 #define PROTO_HTTP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "lib/tls/buffers_tls.h"
 #include "lib/tls/tortls.h"
 #include "lib/compress/compress.h"

src/test/test_channelpadding.c

@@ -21,7 +21,7 @@
 #include "test/log_test_helpers.h"
 #include "lib/tls/tortls.h"
 #include "lib/evloop/timers.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "core/or/cell_st.h"
 #include "feature/nodelist/networkstatus_st.h"

src/test/test_channeltls.c

@@ -8,7 +8,7 @@
 #define TOR_CHANNEL_INTERNAL_
 #include "core/or/or.h"
 #include "lib/net/address.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/channel.h"
 #include "core/or/channeltls.h"
 #include "core/mainloop/connection.h"

src/test/test_extorport.c

@@ -5,7 +5,7 @@
 #define EXT_ORPORT_PRIVATE
 #define MAINLOOP_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/mainloop/connection.h"
 #include "core/or/connection_or.h"
 #include "app/config/config.h"

src/test/test_helpers.c

@@ -14,7 +14,7 @@
 #include "orconfig.h"
 #include "core/or/or.h"
 
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "app/config/confparse.h"
 #include "core/mainloop/connection.h"

src/test/test_oom.c

@@ -8,7 +8,7 @@
 #define CIRCUITLIST_PRIVATE
 #define CONNECTION_PRIVATE
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/circuitlist.h"
 #include "lib/evloop/compat_libevent.h"
 #include "core/mainloop/connection.h"

src/test/test_proto_http.c

@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/proto/proto_http.h"
 #include "test/log_test_helpers.h"
 

src/test/test_proto_misc.c

@@ -8,7 +8,7 @@
 
 #include "core/or/or.h"
 #include "test/test.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "core/or/connection_or.h"
 #include "feature/relay/ext_orport.h"
 #include "core/proto/proto_cell.h"

src/test/test_routerlist.c

@@ -46,7 +46,7 @@
 #include "feature/nodelist/routerstatus_st.h"
 
 #include "lib/encoding/confline.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 
 #include "test/test.h"
 #include "test/test_dir_common.h"

src/test/test_socks.c

@@ -4,7 +4,7 @@
 /* See LICENSE for licensing information */
 
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "core/mainloop/connection.h"
 #include "core/proto/proto_socks.h"

src/test/test_util.c

@@ -13,7 +13,7 @@
 #define SUBPROCESS_PRIVATE
 #include "lib/testsupport/testsupport.h"
 #include "core/or/or.h"
-#include "lib/container/buffers.h"
+#include "lib/buf/buffers.h"
 #include "app/config/config.h"
 #include "feature/control/control.h"
 #include "feature/client/transports.h"