Merge remote-tracking branch 'tor-gitlab/mr/62'

.travis.yml

@@ -237,8 +237,8 @@ install:
   - dd ibs=1 count=1024 if=/dev/urandom > ~/.torrc
 
 script:
-  # Skip test_rebind on macOS
+  # Skip test_rebind and test_include on macOS
-  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export TOR_SKIP_TEST_REBIND=true; fi
+  - if [[ "$TRAVIS_OS_NAME" == "osx" ]]; then export TOR_SKIP_TEST_REBIND=true; export TOR_SKIP_TEST_INCLUDE=true; fi
   - ./autogen.sh
   - CONFIGURE_FLAGS="$ASCIIDOC_OPTIONS $COVERAGE_OPTIONS $HARDENING_OPTIONS $MODULES_OPTIONS $NSS_OPTIONS $OPENSSL_OPTIONS $RUST_OPTIONS --enable-fatal-warnings --disable-silent-rules"
   - echo "Configure flags are $CONFIGURE_FLAGS CC=\"$CC $C_DIALECT_OPTIONS\""

changes/feature25140

@@ -0,0 +1,3 @@
+  o Minor feature (configuration):
+    - Allow the using wildcards (* and ?) with the %include option on 
+      configuration files. Closes ticket 25140. Patch by Daniel Pinto.

configure.ac

@@ -866,6 +866,7 @@ dnl Where do you live, libevent?  And how do we call you?
 if test "$bwin32" = "true"; then
   TOR_LIB_WS32=-lws2_32
   TOR_LIB_IPHLPAPI=-liphlpapi
+  TOR_LIB_SHLWAPI=-lshlwapi
   # Some of the cargo-cults recommend -lwsock32 as well, but I don't
   # think it's actually necessary.
   TOR_LIB_GDI=-lgdi32
@@ -878,6 +879,7 @@ fi
 AC_SUBST(TOR_LIB_WS32)
 AC_SUBST(TOR_LIB_GDI)
 AC_SUBST(TOR_LIB_IPHLPAPI)
+AC_SUBST(TOR_LIB_SHLWAPI)
 AC_SUBST(TOR_LIB_USERENV)
 
 tor_libevent_pkg_redhat="libevent"
@@ -1646,7 +1648,8 @@ AC_CHECK_HEADERS([errno.h \
 		  sys/utime.h \
 		  sys/wait.h \
 		  syslog.h \
-		  utime.h])
+		  utime.h \
+		  glob.h])
 
 AC_CHECK_HEADERS(sys/param.h)
 

doc/man/tor.1.txt

@@ -211,14 +211,22 @@ backslash character (\) before the end of the line.  Comments can be used in
 such multiline entries, but they must start at the beginning of a line.
 
 Configuration options can be imported from files or folders using the %include
-option with the value being a path. If the path is a file, the options from the
+option with the value being a path. This path can have wildcards. Wildcards are 
-file will be parsed as if they were written where the %include option is. If
+expanded first, then sorted using lexical order. Then, for each matching file or 
+folder, the following rules are followed: if the path is a file, the options from 
+the file will be parsed as if they were written where the %include option is. If
 the path is a folder, all files on that folder will be parsed following lexical
-order. Files starting with a dot are ignored. Files on subfolders are ignored.
+order. Files starting with a dot are ignored. Files in subfolders are ignored.
 The %include option can be used recursively.
 New configuration files or directories cannot be added to already running Tor
 instance if **Sandbox** is enabled.
 
+The supported wildcards are * meaning any number of characters including none
+and ? meaning exactly one character. These characters can be escaped by preceding
+them with a backslash, except on Windows. Files starting with a dot are not matched
+when expanding wildcards unless the starting dot is explicitly in the pattern, except
+on Windows.
+
 By default, an option on the command line overrides an option found in the
 configuration file, and an option in a configuration file overrides one in
 the defaults file.

scripts/codegen/fuzzing_include_am.py

@@ -35,7 +35,7 @@ FUZZING_LIBS = \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
 	@TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
-	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
 	@TOR_SYSTEMD_LIBS@ \
 	@TOR_LZMA_LIBS@ \
 	@TOR_ZSTD_LIBS@

src/app/include.am

@@ -18,7 +18,7 @@ src_app_tor_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_li
 src_app_tor_LDADD = $(TOR_INTERNAL_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
-	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
 	@CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
 	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
 
@@ -29,7 +29,7 @@ src_app_tor_cov_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_app_tor_cov_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) @TOR_LDFLAGS_libevent@
 src_app_tor_cov_LDADD = $(TOR_INTERNAL_TESTING_LIBS) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
-	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ \
 	@CURVE25519_LIBS@ @TOR_SYSTEMD_LIBS@ \
 	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
 endif

src/config/torrc.sample.in

@@ -242,11 +242,12 @@
 #PublishServerDescriptor 0
 
 ## Configuration options can be imported from files or folders using the %include
-## option with the value being a path. If the path is a file, the options from the
+## option with the value being a path. This path can have wildcards. Wildcards are 
-## file will be parsed as if they were written where the %include option is. If
+## expanded first, using lexical order. Then, for each matching file or folder, the following 
-## the path is a folder, all files on that folder will be parsed following lexical
+## rules are followed: if the path is a file, the options from the file will be parsed as if 
-## order. Files starting with a dot are ignored. Files on subfolders are ignored.
+## they were written where the %include option is. If the path is a folder, all files on that 
+## folder will be parsed following lexical order. Files starting with a dot are ignored. Files 
+## on subfolders are ignored.
 ## The %include option can be used recursively.
-#%include /etc/torrc.d/
+#%include /etc/torrc.d/*.conf
-#%include /etc/torrc.custom
 

src/lib/fs/conffile.c

@@ -21,6 +21,8 @@
 #include "lib/malloc/malloc.h"
 #include "lib/string/printf.h"
 
+#include <stdbool.h>
+
 static smartlist_t *config_get_file_list(const char *path,
                                          smartlist_t *opened_files);
 static int config_get_included_config(const char *path, int recursion_level,
@@ -50,62 +52,109 @@ config_get_lines_include(const char *string, config_line_t **result,
                               opened_lst, 1, NULL, config_process_include);
 }
 
-/** Adds a list of configuration files present on <b>path</b> to
+/** Returns a list of paths obtained when expading globs in <b>pattern</b>. If
- * <b>file_list</b>. <b>path</b> can be a file or a directory. If it is a file,
+ * <b>pattern</b> has no globs, returns a list with <b>pattern</b> if it is an
- * only that file will be added to <b>file_list</b>. If it is a directory,
+ * existing path or NULL otherwise. If <b>opened_files</b> is provided, adds
- * all paths for files on that directory root (no recursion) except for files
+ * paths opened by glob to it. Returns NULL on failure. */
- * whose name starts with a dot will be added to <b>file_list</b>.
- * <b>opened_files</b> will have a list of files opened by this function
- * if provided. Return 0 on success, -1 on failure. Ignores empty files.
- */
 static smartlist_t *
-config_get_file_list(const char *path, smartlist_t *opened_files)
+expand_glob(const char *pattern, smartlist_t *opened_files)
 {
-  smartlist_t *file_list = smartlist_new();
+  smartlist_t *matches = tor_glob(pattern);
-
+  if (!matches) {
-  if (opened_files) {
-    smartlist_add_strdup(opened_files, path);
-  }
-
-  file_status_t file_type = file_status(path);
-  if (file_type == FN_FILE) {
-    smartlist_add_strdup(file_list, path);
-    return file_list;
-  } else if (file_type == FN_DIR) {
-    smartlist_t *all_files = tor_listdir(path);
-    if (!all_files) {
-      smartlist_free(file_list);
-      return NULL;
-    }
-    smartlist_sort_strings(all_files);
-    SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
-      if (f[0] == '.') {
-        tor_free(f);
-        continue;
-      }
-
-      char *fullname;
-      tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
-      tor_free(f);
-
-      if (opened_files) {
-        smartlist_add_strdup(opened_files, fullname);
-      }
-
-      if (file_status(fullname) != FN_FILE) {
-        tor_free(fullname);
-        continue;
-      }
-      smartlist_add(file_list, fullname);
-    } SMARTLIST_FOREACH_END(f);
-    smartlist_free(all_files);
-    return file_list;
-  } else if (file_type == FN_EMPTY) {
-      return file_list;
-  } else {
-    smartlist_free(file_list);
     return NULL;
   }
+
+  // if it is not a glob, return error when the path is missing
+  if (!has_glob(pattern) && smartlist_len(matches) == 0) {
+    smartlist_free(matches);
+    return NULL;
+  }
+
+  if (opened_files) {
+    smartlist_t *glob_opened = get_glob_opened_files(pattern);
+    if (!glob_opened) {
+      SMARTLIST_FOREACH(matches, char *, f, tor_free(f));
+      smartlist_free(matches);
+      return NULL;
+    }
+    smartlist_add_all(opened_files, glob_opened);
+    smartlist_free(glob_opened);
+  }
+  smartlist_sort_strings(matches);
+  return matches;
+}
+
+/** Returns a list of configuration files present on paths that match
+ * <b>pattern</b>. The pattern is expanded and then all the paths are
+ * processed. A path can be a file or a directory. If it is a file, that file
+ * will be added to the list to be returned. If it is a directory,
+ * all paths for files on that directory root (no recursion) except for files
+ * whose name starts with a dot will be added to the list to be returned.
+ * <b>opened_files</b> will have a list of files opened by this function
+ * if provided. Return NULL on failure. Ignores empty files.
+ */
+static smartlist_t *
+config_get_file_list(const char *pattern, smartlist_t *opened_files)
+{
+  smartlist_t *glob_matches = expand_glob(pattern, opened_files);
+  if (!glob_matches) {
+    return NULL;
+  }
+
+  bool error_found = false;
+  smartlist_t *file_list = smartlist_new();
+  SMARTLIST_FOREACH_BEGIN(glob_matches, char *, path) {
+    if (opened_files) {
+      smartlist_add_strdup(opened_files, path);
+    }
+
+    file_status_t file_type = file_status(path);
+    if (file_type == FN_FILE) {
+      smartlist_add_strdup(file_list, path);
+    } else if (file_type == FN_DIR) {
+      smartlist_t *all_files = tor_listdir(path);
+      if (!all_files) {
+        error_found = true;
+        break;
+      }
+      smartlist_sort_strings(all_files);
+      SMARTLIST_FOREACH_BEGIN(all_files, char *, f) {
+        if (f[0] == '.') {
+          continue;
+        }
+
+        char *fullname;
+        tor_asprintf(&fullname, "%s"PATH_SEPARATOR"%s", path, f);
+
+        if (opened_files) {
+          smartlist_add_strdup(opened_files, fullname);
+        }
+
+        if (file_status(fullname) != FN_FILE) {
+          tor_free(fullname);
+          continue;
+        }
+        smartlist_add(file_list, fullname);
+      } SMARTLIST_FOREACH_END(f);
+      SMARTLIST_FOREACH(all_files, char *, f, tor_free(f));
+      smartlist_free(all_files);
+    } else if (file_type == FN_EMPTY) {
+        continue;
+    } else {
+      error_found = true;
+      break;
+    }
+  } SMARTLIST_FOREACH_END(path);
+  SMARTLIST_FOREACH(glob_matches, char *, f, tor_free(f));
+  smartlist_free(glob_matches);
+
+  if (error_found) {
+    SMARTLIST_FOREACH(file_list, char *, f, tor_free(f));
+    smartlist_free(file_list);
+    file_list = NULL;
+  }
+
+  return file_list;
 }
 
 /** Creates a list of config lines present on included <b>path</b>.
@@ -133,19 +182,19 @@ config_get_included_config(const char *path, int recursion_level, int extended,
   return 0;
 }
 
-/** Process an %include <b>path</b> in a config file. Set <b>list</b> to the
+/** Process an %include <b>pattern</b> in a config file. Set <b>list</b> to the
  * list of configuration settings obtained and <b>list_last</b> to the last
  * element of the same list. <b>opened_lst</b> will have a list of opened
  * files if provided. Return 0 on success, -1 on failure. */
 static int
-config_process_include(const char *path, int recursion_level, int extended,
+config_process_include(const char *pattern, int recursion_level, int extended,
                        config_line_t **list, config_line_t **list_last,
                        smartlist_t *opened_lst)
 {
   config_line_t *ret_list = NULL;
   config_line_t **next = &ret_list;
 
-  smartlist_t *config_files = config_get_file_list(path, opened_lst);
+  smartlist_t *config_files = config_get_file_list(pattern, opened_lst);
   if (!config_files) {
     return -1;
   }

src/lib/fs/files.c

@@ -247,6 +247,22 @@ file_status(const char *fname)
   }
 }
 
+/** Returns true if <b>file_type</b> represents an existing file (even if
+ * empty). Returns false otherwise. */
+bool
+is_file(file_status_t file_type)
+{
+  return file_type != FN_ERROR && file_type != FN_NOENT && file_type != FN_DIR;
+}
+
+/** Returns true if <b>file_type</b> represents an existing directory. Returns
+ * false otherwise. */
+bool
+is_dir(file_status_t file_type)
+{
+  return file_type == FN_DIR;
+}
+
 /** Create a file named <b>fname</b> with the contents <b>str</b>.  Overwrite
  * the previous <b>fname</b> if possible.  Return 0 on success, -1 on failure.
  *

src/lib/fs/files.h

@@ -55,6 +55,8 @@ MOCK_DECL(int,tor_unlink,(const char *pathname));
 typedef enum { FN_ERROR, FN_NOENT, FN_FILE, FN_DIR, FN_EMPTY } file_status_t;
 
 file_status_t file_status(const char *filename);
+bool is_file(file_status_t file_type);
+bool is_dir(file_status_t file_type);
 
 int64_t tor_get_avail_disk_space(const char *path);
 

src/lib/fs/path.c

@@ -13,15 +13,34 @@
 #include "lib/malloc/malloc.h"
 #include "lib/log/log.h"
 #include "lib/log/util_bug.h"
+#include "lib/container/smartlist.h"
+#include "lib/sandbox/sandbox.h"
 #include "lib/string/printf.h"
 #include "lib/string/util_string.h"
 #include "lib/string/compat_ctype.h"
+#include "lib/string/compat_string.h"
+#include "lib/fs/files.h"
+#include "lib/fs/dir.h"
 #include "lib/fs/userdb.h"
 
+#ifdef HAVE_SYS_TYPES_H
+#include <sys/types.h>
+#endif
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
 #ifdef HAVE_UNISTD_H
 #include <unistd.h>
 #endif
 
+#ifdef _WIN32
+#include <windows.h>
+#include <shlwapi.h>
+#else /* !(defined(_WIN32)) */
+#include <dirent.h>
+#include <glob.h>
+#endif /* defined(_WIN32) */
+
 #include <errno.h>
 #include <string.h>
 
@@ -294,3 +313,346 @@ make_path_absolute(const char *fname)
   return absfname;
 #endif /* defined(_WIN32) */
 }
+
+/* The code below implements tor_glob and get_glob_opened_files. Because it is
+ * not easy to understand it by looking at individual functions, the big
+ * picture explanation here should be read first.
+ *
+ * Purpose of the functions:
+ * - tor_glob - recevies a pattern and returns all the paths that result from
+ *   its glob expansion, globs can be present on all path components.
+ * - get_glob_opened_files - receives a pattern and returns all the paths that
+ *   are opened during its expansion (the paths before any path fragment that
+ *   contains a glob as they have to be opened to check for glob matches). This
+ *   is used to get the paths that have to be added to the seccomp sandbox
+ *   allowed list.
+ *
+ * Due to OS API differences explained below, the implementation of tor_glob is
+ * completly different for Windows and POSIX systems, so we ended up with three
+ * different implementations:
+ * - tor_glob for POSIX - as POSIX glob does everything we need, we simply call
+ *   it and process the results. This is completly implemented in tor_glob.
+ * - tor_glob for WIN32 - because the WIN32 API only supports expanding globs
+ *   in the last path fragment, we need to expand the globs in each path
+ *   fragment manually and call recursively to get the same behaviour as POSIX
+ *   glob. When there are no globs in pattern, we know we are on the last path
+ *   fragment and collect the full path.
+ * - get_glob_opened_files - because the paths before any path fragment with a
+ *   glob will be opened to check for matches, we need to collect them and we
+ *   need to expand the globs in each path fragments and call recursively until
+ *   we find no more globs.
+ *
+ * As seen from the description above, both tor_glob for WIN32 and
+ * get_glob_opened_files receive a pattern and return a list of paths and have
+ * to expand all path fragments that contain globs and call themselves
+ * recursively. The differences are:
+ * - get_glob_opened_files collects paths before path fragments with globs
+ *   while tor_glob for WIN32 collects full paths resulting from the expansion
+ *   of all globs.
+ * - get_glob_opened_files can call tor_glob to expand path fragments with
+ *   globs while tor_glob for WIN32 cannot because it IS tor_glob. For tor_glob
+ *   for WIN32, an auxiliary function has to be used for this purpose.
+ *
+ * To avoid code duplication, the logic of tor_glob for WIN32 and
+ * get_glob_opened_files is implemented in get_glob_paths. The differences are
+ * configured by the extra function parameters:
+ * - final - if true, returns a list of paths obtained from expanding pattern
+ *   (implements tor_glob). Otherwise, returns the paths before path fragments
+ *   with globs (implements get_glob_opened_files).
+ * - unglob - function used to expand a path fragment. The function signature
+ *   is defined by the unglob_fn typedef. Two implementations are available:
+ *   - unglob_win32 - uses tor_listdir and PathMatchSpec (for tor_glob WIN32)
+ *   - unglob_opened_files - uses tor_glob (for get_glob_opened_files)
+ */
+
+/** Returns true if the character at position <b>pos</b> in <b>pattern</b> is
+ * considered a glob. Returns false otherwise. Takes escaping into account on
+ * systems where escaping globs is supported. */
+static inline bool
+is_glob_char(const char *pattern, int pos)
+{
+  bool is_glob = pattern[pos] == '*' || pattern[pos] == '?';
+#ifdef _WIN32
+  return is_glob;
+#else /* !defined(_WIN32) */
+  bool is_escaped = pos > 0 && pattern[pos-1] == '\\';
+  return is_glob && !is_escaped;
+#endif /* defined(_WIN32) */
+}
+
+/** Expands the first path fragment of <b>pattern</b> that contains globs. The
+ * path fragment is between <b>prev_sep</b> and <b>next_sep</b>. If the path
+ * fragment is the last fragment of <b>pattern</b>, <b>next_sep</b> will be the
+ * index of the last char. Returns a list of paths resulting from the glob
+ * expansion of the path fragment. Anything after <b>next_sep</b> is not
+ * included in the returned list. Returns NULL on failure. */
+typedef struct smartlist_t * unglob_fn(const char *pattern, int prev_sep,
+                                       int next_sep);
+
+/** Adds <b>path</b> to <b>result</b> if it exists and is a file type we can
+ * handle. Returns false if <b>path</b> is a file type we cannot handle,
+ * returns true otherwise. Used on tor_glob for WIN32. */
+static bool
+add_non_glob_path(const char *path, struct smartlist_t *result)
+{
+  file_status_t file_type = file_status(path);
+  if (file_type == FN_ERROR) {
+    return false;
+  } else if (file_type != FN_NOENT) {
+    char *to_add = tor_strdup(path);
+    clean_fname_for_stat(to_add);
+    smartlist_add(result, to_add);
+  }
+  /* If WIN32 tor_glob is called with a non-existing path, we want it to
+   * return an empty list instead of error to match the regular version */
+  return true;
+}
+
+/** Auxiliary function used by get_glob_opened_files and WIN32 tor_glob.
+ * Returns a list of paths obtained from <b>pattern</b> using <b>unglob</b> to
+ * expand each path fragment. If <b>final</b> is true, the paths are the result
+ * of the glob expansion of <b>pattern</b> (implements tor_glob). Otherwise,
+ * the paths are the paths opened by glob while expanding <b>pattern</b>
+ * (implements get_glob_opened_files). Returns NULL on failure. */
+static struct smartlist_t *
+get_glob_paths(const char *pattern, unglob_fn unglob, bool final)
+{
+  smartlist_t *result = smartlist_new();
+  int i, prev_sep = -1, next_sep = -1;
+  bool is_glob = false, error_found = false, is_sep = false, is_last = false;
+
+  // find first path fragment with globs
+  for (i = 0; pattern[i]; i++) {
+    is_glob = is_glob || is_glob_char(pattern, i);
+    is_last = !pattern[i+1];
+    is_sep = pattern[i] == *PATH_SEPARATOR || pattern[i] == '/';
+    if (is_sep || is_last) {
+      prev_sep = next_sep;
+      next_sep = i; // next_sep+1 is start of next fragment or end of string
+      if (is_glob) {
+        break;
+      }
+    }
+  }
+
+  if (!is_glob) { // pattern fully expanded or no glob in pattern
+    if (final && !add_non_glob_path(pattern, result)) {
+      error_found = true;
+      goto end;
+    }
+    return result;
+  }
+
+  if (!final) {
+    // add path before the glob to result
+    int len = prev_sep < 1 ? prev_sep + 1 : prev_sep; // handle /*
+    char *path_until_glob = tor_strndup(pattern, len);
+    smartlist_add(result, path_until_glob);
+  }
+
+  smartlist_t *unglobbed_paths = unglob(pattern, prev_sep, next_sep);
+  if (!unglobbed_paths) {
+    error_found = true;
+  } else {
+    // for each path for current fragment, add the rest of the pattern
+    // and call recursively to get all expanded paths
+    SMARTLIST_FOREACH_BEGIN(unglobbed_paths, char *, current_path) {
+      char *next_path;
+      tor_asprintf(&next_path, "%s"PATH_SEPARATOR"%s", current_path,
+                   &pattern[next_sep+1]);
+      smartlist_t *opened_next = get_glob_paths(next_path, unglob, final);
+      tor_free(next_path);
+      if (!opened_next) {
+        error_found = true;
+        break;
+      }
+      smartlist_add_all(result, opened_next);
+      smartlist_free(opened_next);
+    } SMARTLIST_FOREACH_END(current_path);
+    SMARTLIST_FOREACH(unglobbed_paths, char *, p, tor_free(p));
+    smartlist_free(unglobbed_paths);
+  }
+
+end:
+  if (error_found) {
+    SMARTLIST_FOREACH(result, char *, p, tor_free(p));
+    smartlist_free(result);
+    result = NULL;
+  }
+  return result;
+}
+
+#ifdef _WIN32
+/** Expands globs in <b>pattern</b> for the path fragment between
+ * <b>prev_sep</b> and <b>next_sep</b> using the WIN32 API. Returns NULL on
+ * failure. Used by the WIN32 implementation of tor_glob. Implements unglob_fn,
+ * see its description for more details. */
+static struct smartlist_t *
+unglob_win32(const char *pattern, int prev_sep, int next_sep)
+{
+  smartlist_t *result = smartlist_new();
+  int len = prev_sep < 1 ? prev_sep + 1 : prev_sep; // handle /*
+  char *path_until_glob = tor_strndup(pattern, len);
+
+  if (!is_file(file_status(path_until_glob))) {
+    smartlist_t *filenames = tor_listdir(path_until_glob);
+    if (!filenames) {
+      smartlist_free(result);
+      result = NULL;
+    } else {
+      SMARTLIST_FOREACH_BEGIN(filenames, char *, filename) {
+        TCHAR tpattern[MAX_PATH] = {0};
+        TCHAR tfile[MAX_PATH] = {0};
+        char *full_path;
+        tor_asprintf(&full_path, "%s"PATH_SEPARATOR"%s",
+                     path_until_glob, filename);
+        char *path_curr_glob = tor_strndup(pattern, next_sep + 1);
+        // *\ must return only dirs, remove \ from the pattern so it matches
+        if (is_dir(file_status(full_path))) {
+          clean_fname_for_stat(path_curr_glob);
+        }
+#ifdef UNICODE
+        mbstowcs(tpattern, path_curr_glob, MAX_PATH);
+        mbstowcs(tfile, full_path, MAX_PATH);
+#else /* !defined(UNICODE) */
+        strlcpy(tpattern, path_curr_glob, MAX_PATH);
+        strlcpy(tfile, full_path, MAX_PATH);
+#endif /* defined(UNICODE) */
+        if (PathMatchSpec(tfile, tpattern)) {
+          smartlist_add(result, full_path);
+        } else {
+          tor_free(full_path);
+        }
+        tor_free(path_curr_glob);
+      } SMARTLIST_FOREACH_END(filename);
+      SMARTLIST_FOREACH(filenames, char *, p, tor_free(p));
+      smartlist_free(filenames);
+    }
+  }
+  tor_free(path_until_glob);
+  return result;
+}
+#else /* !defined(_WIN32) */
+/** Same as opendir but calls sandbox_intern_string before */
+static DIR *
+prot_opendir(const char *name)
+{
+  return opendir(sandbox_intern_string(name));
+}
+
+/** Same as stat but calls sandbox_intern_string before */
+static int
+prot_stat(const char *pathname, struct stat *buf)
+{
+  return stat(sandbox_intern_string(pathname), buf);
+}
+
+/** Same as lstat but calls sandbox_intern_string before */
+static int
+prot_lstat(const char *pathname, struct stat *buf)
+{
+  return lstat(sandbox_intern_string(pathname), buf);
+}
+#endif /* defined(_WIN32) */
+
+/** Return a new list containing the paths that match the pattern
+ * <b>pattern</b>. Return NULL on error. On POSIX systems, errno is set by the
+ * glob function.
+ */
+struct smartlist_t *
+tor_glob(const char *pattern)
+{
+  smartlist_t *result;
+#ifdef _WIN32
+  // PathMatchSpec does not support forward slashes, change them to backslashes
+  char *pattern_normalized = tor_strdup(pattern);
+  tor_strreplacechar(pattern_normalized, '/', *PATH_SEPARATOR);
+  result = get_glob_paths(pattern_normalized, unglob_win32, true);
+  tor_free(pattern_normalized);
+#else /* !(defined(_WIN32)) */
+  glob_t matches;
+  int flags = GLOB_ERR | GLOB_NOSORT;
+#ifdef GLOB_ALTDIRFUNC
+  /* use functions that call sandbox_intern_string */
+  flags |= GLOB_ALTDIRFUNC;
+  typedef void *(*gl_opendir)(const char * name);
+  typedef struct dirent *(*gl_readdir)(void *);
+  typedef void (*gl_closedir)(void *);
+  matches.gl_opendir = (gl_opendir) &prot_opendir;
+  matches.gl_readdir = (gl_readdir) &readdir;
+  matches.gl_closedir = (gl_closedir) &closedir;
+  matches.gl_stat = &prot_stat;
+  matches.gl_lstat = &prot_lstat;
+#endif /* defined(GLOB_ALTDIRFUNC) */
+  int ret = glob(pattern, flags, NULL, &matches);
+  if (ret == GLOB_NOMATCH) {
+    return smartlist_new();
+  } else if (ret != 0) {
+    return NULL;
+  }
+
+  result = smartlist_new();
+  size_t i;
+  for (i = 0; i < matches.gl_pathc; i++) {
+    char *match = tor_strdup(matches.gl_pathv[i]);
+    size_t len = strlen(match);
+    if (len > 0 && match[len-1] == *PATH_SEPARATOR) {
+      match[len-1] = '\0';
+    }
+    smartlist_add(result, match);
+  }
+  globfree(&matches);
+#endif /* defined(_WIN32) */
+  return result;
+}
+
+/** Returns true if <b>s</b> contains characters that can be globbed.
+ * Returns false otherwise. */
+bool
+has_glob(const char *s)
+{
+  int i;
+  for (i = 0; s[i]; i++) {
+    if (is_glob_char(s, i)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+/** Expands globs in <b>pattern</b> for the path fragment between
+ * <b>prev_sep</b> and <b>next_sep</b> using tor_glob. Returns NULL on
+ * failure. Used by get_glob_opened_files. Implements unglob_fn, see its
+ * description for more details. */
+static struct smartlist_t *
+unglob_opened_files(const char *pattern, int prev_sep, int next_sep)
+{
+  (void)prev_sep;
+  smartlist_t *result = smartlist_new();
+  // if the following fragments have no globs, we're done
+  if (has_glob(&pattern[next_sep+1])) {
+    // if there is a glob after next_sep, we know next_sep is a separator and
+    // not the last char and glob_path will have the path without the separator
+    char *glob_path = tor_strndup(pattern, next_sep);
+    smartlist_t *child_paths = tor_glob(glob_path);
+    tor_free(glob_path);
+    if (!child_paths) {
+      smartlist_free(result);
+      result = NULL;
+    } else {
+      smartlist_add_all(result, child_paths);
+      smartlist_free(child_paths);
+    }
+  }
+  return result;
+}
+
+/** Returns a list of files that are opened by the tor_glob function when
+ * called with <b>pattern</b>. Returns NULL on error. The purpose of this
+ * function is to create a list of files to be added to the sandbox white list
+ * before the sandbox is enabled. */
+struct smartlist_t *
+get_glob_opened_files(const char *pattern)
+{
+  return get_glob_paths(pattern, unglob_opened_files, false);
+}

src/lib/fs/path.h

@@ -12,6 +12,10 @@
 #ifndef TOR_PATH_H
 #define TOR_PATH_H
 
+#include <stdbool.h>
+#ifdef _WIN32
+#include <windows.h>
+#endif
 #include "lib/cc/compat_compiler.h"
 
 #ifdef _WIN32
@@ -26,5 +30,8 @@ int path_is_relative(const char *filename);
 void clean_fname_for_stat(char *name);
 int get_parent_directory(char *fname);
 char *make_path_absolute(const char *fname);
+struct smartlist_t *tor_glob(const char *pattern);
+bool has_glob(const char *s);
+struct smartlist_t *get_glob_opened_files(const char *pattern);
 
 #endif /* !defined(TOR_PATH_H) */

src/lib/sandbox/sandbox.c

@@ -204,6 +204,8 @@ static int filter_nopar_gen[] = {
 #ifdef __NR__llseek
     SCMP_SYS(_llseek),
 #endif
+    // glob uses this..
+    SCMP_SYS(lstat),
     SCMP_SYS(mkdir),
     SCMP_SYS(mlockall),
 #ifdef __NR_mmap

src/lib/string/util_string.c

@@ -143,6 +143,15 @@ tor_strupper(char *s)
   }
 }
 
+/** Replaces <b>old</b> with <b>replacement</b> in <b>s</b> */
+void
+tor_strreplacechar(char *s, char find, char replacement)
+{
+  for (s = strchr(s, find); s; s = strchr(s + 1, find)) {
+    *s = replacement;
+  }
+}
+
 /** Return 1 if every character in <b>s</b> is printable, else return 0.
  */
 int

src/lib/string/util_string.h

@@ -31,6 +31,7 @@ int tor_digest256_is_zero(const char *digest);
 #define HEX_CHARACTERS "0123456789ABCDEFabcdef"
 void tor_strlower(char *s);
 void tor_strupper(char *s);
+void tor_strreplacechar(char *s, char find, char replacement);
 int tor_strisprint(const char *s);
 int tor_strisnonupper(const char *s);
 int tor_strisspace(const char *s);

src/test/fuzz/include.am

@@ -11,7 +11,7 @@ FUZZING_LIBS = \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
 	@TOR_LIBEVENT_LIBS@ $(TOR_LIBS_CRYPTLIB) \
-	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@ \
 	@TOR_SYSTEMD_LIBS@ \
 	@TOR_LZMA_LIBS@ \
 	@TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@

src/test/include.am

@@ -46,6 +46,7 @@ else
 # Only do this when coverage is not on, since it invokes lots of code
 # in a kind of unpredictable way.
 TESTSCRIPTS += src/test/test_rebind.sh
+TESTSCRIPTS += src/test/test_include.sh
 endif
 endif
 
@@ -296,16 +297,15 @@ src_test_test_switch_id_LDADD = \
 	$(TOR_UTIL_TESTING_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_USERENV@ \
 	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
-
 src_test_test_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB) \
 	@TOR_LDFLAGS_libevent@
 src_test_test_LDADD = \
 	$(TOR_INTERNAL_TESTING_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
 	@CURVE25519_LIBS@ \
 	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
 
@@ -334,7 +334,7 @@ src_test_bench_LDADD = \
 	$(TOR_INTERNAL_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
 	@CURVE25519_LIBS@ \
 	@TOR_SYSTEMD_LIBS@ @TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
 
@@ -344,7 +344,7 @@ src_test_test_workqueue_LDADD = \
 	$(TOR_INTERNAL_TESTING_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
 	@CURVE25519_LIBS@ \
 	@TOR_LZMA_LIBS@ @TOR_ZSTD_LIBS@ @TOR_TRACE_LIBS@
 
@@ -356,7 +356,7 @@ src_test_test_timers_LDADD = \
 	$(TOR_UTIL_TESTING_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ @TOR_LIBEVENT_LIBS@ \
-	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
 	@CURVE25519_LIBS@ \
 	@TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@
 src_test_test_timers_LDFLAGS = $(src_test_test_LDFLAGS)
@@ -393,7 +393,7 @@ src_test_test_ntor_cl_LDADD = \
 	$(TOR_INTERNAL_LIBS) \
 	$(rust_ldadd) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
 	@CURVE25519_LIBS@ @TOR_LZMA_LIBS@ @TOR_TRACE_LIBS@
 src_test_test_ntor_cl_AM_CPPFLAGS =	       \
 	$(AM_CPPFLAGS)
@@ -403,8 +403,8 @@ src_test_test_hs_ntor_cl_LDFLAGS = @TOR_LDFLAGS_zlib@ $(TOR_LDFLAGS_CRYPTLIB)
 src_test_test_hs_ntor_cl_LDADD = \
 	$(TOR_INTERNAL_LIBS) \
 	@TOR_ZLIB_LIBS@ @TOR_LIB_MATH@ \
-	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ \
+	$(TOR_LIBS_CRYPTLIB) @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ \
-	@CURVE25519_LIBS@ @TOR_TRACE_LIBS@
+        @CURVE25519_LIBS@ @TOR_TRACE_LIBS@
 src_test_test_hs_ntor_cl_AM_CPPFLAGS =	       \
 	$(AM_CPPFLAGS)
 
@@ -416,8 +416,8 @@ src_test_test_bt_cl_LDADD = \
 	$(TOR_UTIL_TESTING_LIBS) \
 	$(rust_ldadd) \
 	@TOR_LIB_MATH@ \
-	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ \
-	@TOR_TRACE_LIBS@
+        @TOR_TRACE_LIBS@
 src_test_test_bt_cl_CFLAGS = $(AM_CFLAGS) $(TEST_CFLAGS)
 src_test_test_bt_cl_CPPFLAGS= $(src_test_AM_CPPFLAGS) $(TEST_CPPFLAGS)
 endif
@@ -432,6 +432,8 @@ EXTRA_DIST += \
 	src/test/slownacl_curve25519.py \
 	src/test/test_rebind.sh \
 	src/test/test_rebind.py \
+	src/test/test_include.sh \
+	src/test/test_include.py \
 	src/test/zero_length_keys.sh \
 	scripts/maint/run_check_subsystem_order.sh \
 	src/test/rust_supp.txt \

src/test/test_config.c

@@ -5889,6 +5889,7 @@ test_config_include_flag_both_without(void *data)
 
  done:
   tor_free(errmsg);
+  config_free_all();
 }
 
 static void
@@ -5929,6 +5930,7 @@ test_config_include_flag_torrc_only(void *data)
   tor_free(errmsg);
   tor_free(path);
   tor_free(dir);
+  config_free_all();
 }
 
 static void
@@ -5969,6 +5971,287 @@ test_config_include_flag_defaults_only(void *data)
   tor_free(errmsg);
   tor_free(path);
   tor_free(dir);
+  config_free_all();
+}
+
+static void
+test_config_include_wildcards(void *data)
+{
+  (void)data;
+
+  char *temp = NULL, *folder = NULL;
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_wildcards"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", dir, "01_one.conf");
+  tt_int_op(write_str_to_file(temp, "Test 1\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", dir, "02_two.conf");
+  tt_int_op(write_str_to_file(temp, "Test 2\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", dir, "aa_three.conf");
+  tt_int_op(write_str_to_file(temp, "Test 3\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", dir, "foo");
+  tt_int_op(write_str_to_file(temp, "Test 6\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  tor_asprintf(&folder, "%s"PATH_SEPARATOR"%s", dir, "folder");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(folder), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(folder, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", folder, "04_four.conf");
+  tt_int_op(write_str_to_file(temp, "Test 4\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", folder, "05_five.conf");
+  tt_int_op(write_str_to_file(temp, "Test 5\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  char torrc_contents[1000];
+  int include_used;
+
+  // test pattern that matches no file
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"not-exist*\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+  config_free_lines(result);
+
+#ifndef _WIN32
+  // test wildcard escaping
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"\\*\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, -1);
+  tt_ptr_op(result, OP_EQ, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+  config_free_lines(result);
+#endif
+
+  // test pattern *.conf
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"*.conf\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  int len = 0;
+  config_line_t *next;
+  char expected[10];
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 3);
+  config_free_lines(result);
+
+  // test pattern that matches folder and files
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"*\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  len = 0;
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 6);
+  config_free_lines(result);
+
+  // test pattern ending in PATH_SEPARATOR, test linux path separator
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s/f*/\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  len = 0;
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1 + 3);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 2);
+  config_free_lines(result);
+
+  // test pattern with wildcards in folder and file
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"*"PATH_SEPARATOR"*.conf\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  len = 0;
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1 + 3);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 2);
+  config_free_lines(result);
+
+ done:
+  config_free_lines(result);
+  tor_free(folder);
+  tor_free(temp);
+  tor_free(dir);
+}
+
+static void
+test_config_include_hidden(void *data)
+{
+  (void)data;
+
+  char *temp = NULL, *folder = NULL;
+  config_line_t *result = NULL;
+  char *dir = tor_strdup(get_fname("test_include_hidden"));
+  tt_ptr_op(dir, OP_NE, NULL);
+
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&folder, "%s"PATH_SEPARATOR"%s", dir, ".dotdir");
+
+#ifdef _WIN32
+  tt_int_op(mkdir(folder), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(folder, 0700), OP_EQ, 0);
+#endif
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", folder, ".dotfile");
+  tt_int_op(write_str_to_file(temp, "Test 1\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  tor_asprintf(&temp, "%s"PATH_SEPARATOR"%s", folder, "file");
+  tt_int_op(write_str_to_file(temp, "Test 2\n", 0), OP_EQ, 0);
+  tor_free(temp);
+
+  char torrc_contents[1000];
+  int include_used;
+  int len = 0;
+  config_line_t *next;
+  char expected[10];
+
+  // test wildcards do not expand to dot folders (except for windows)
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"*\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_int_op(include_used, OP_EQ, 1);
+#ifdef _WIN32 // wildcard expansion includes dot files on Windows
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 2);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 1);
+#else
+  tt_ptr_op(result, OP_EQ, NULL);
+#endif
+  config_free_lines(result);
+
+  // test wildcards match hidden folders when explicitly in the pattern
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR".*\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  len = 0;
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 2);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 1);
+  config_free_lines(result);
+
+  // test hidden dir when explicitly included
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR".dotdir\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  len = 0;
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 2);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 1);
+  config_free_lines(result);
+
+  // test hidden file when explicitly included
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR".dotdir"PATH_SEPARATOR".dotfile\n",
+               dir);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            NULL), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+  len = 0;
+  for (next = result; next != NULL; next = next->next) {
+    tor_snprintf(expected, sizeof(expected), "%d", len + 1);
+    tt_str_op(next->key, OP_EQ, "Test");
+    tt_str_op(next->value, OP_EQ, expected);
+    len++;
+  }
+  tt_int_op(len, OP_EQ, 1);
+  config_free_lines(result);
+
+ done:
+  config_free_lines(result);
+  tor_free(folder);
+  tor_free(temp);
+  tor_free(dir);
 }
 
 static void
@@ -6100,7 +6383,7 @@ test_config_include_opened_file_list(void *data)
   smartlist_t *opened_files = smartlist_new();
   char *torrcd = NULL;
   char *subfolder = NULL;
-  char *path = NULL;
+  char *in_subfolder = NULL;
   char *empty = NULL;
   char *file = NULL;
   char *dot = NULL;
@@ -6129,9 +6412,9 @@ test_config_include_opened_file_list(void *data)
   tt_int_op(mkdir(subfolder, 0700), OP_EQ, 0);
 #endif
 
-  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", subfolder,
+  tor_asprintf(&in_subfolder, "%s"PATH_SEPARATOR"%s", subfolder,
                "01_file_in_subfolder");
-  tt_int_op(write_str_to_file(path, "Test 1\n", 0), OP_EQ, 0);
+  tt_int_op(write_str_to_file(in_subfolder, "Test 1\n", 0), OP_EQ, 0);
 
   tor_asprintf(&empty, "%s"PATH_SEPARATOR"%s", torrcd, "empty");
   tt_int_op(write_str_to_file(empty, "", 0), OP_EQ, 0);
@@ -6162,13 +6445,69 @@ test_config_include_opened_file_list(void *data)
   // dot files are not opened as we ignore them when we get their name from
   // their parent folder
 
+  // test with wildcards
+  SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
+  smartlist_clear(opened_files);
+  config_free_lines(result);
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"*\n",
+               torrcd);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            opened_files), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+#ifdef _WIN32
+  tt_int_op(smartlist_len(opened_files), OP_EQ, 6);
+#else
+  tt_int_op(smartlist_len(opened_files), OP_EQ, 5);
+#endif
+  tt_int_op(smartlist_contains_string(opened_files, torrcd), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, subfolder), OP_EQ, 1);
+  // * will match the subfolder inside torrc.d, so it will be included
+  tt_int_op(smartlist_contains_string(opened_files, in_subfolder), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, empty), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, file), OP_EQ, 1);
+#ifdef _WIN32
+  // * matches the dot file on Windows
+  tt_int_op(smartlist_contains_string(opened_files, dot), OP_EQ, 1);
+#endif
+
+  // test with wildcards in folder and file
+  SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
+  smartlist_clear(opened_files);
+  config_free_lines(result);
+  tor_snprintf(torrc_contents, sizeof(torrc_contents),
+               "%%include %s"PATH_SEPARATOR"*"PATH_SEPARATOR"*\n",
+               torrcd);
+  tt_int_op(config_get_lines_include(torrc_contents, &result, 0, &include_used,
+            opened_files), OP_EQ, 0);
+  tt_ptr_op(result, OP_NE, NULL);
+  tt_int_op(include_used, OP_EQ, 1);
+
+#ifdef _WIN32
+  tt_int_op(smartlist_len(opened_files), OP_EQ, 6);
+#else
+  tt_int_op(smartlist_len(opened_files), OP_EQ, 5);
+#endif
+  tt_int_op(smartlist_contains_string(opened_files, torrcd), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, subfolder), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, in_subfolder), OP_EQ, 1);
+  // stat is called on the following files, so they count as opened
+  tt_int_op(smartlist_contains_string(opened_files, empty), OP_EQ, 1);
+  tt_int_op(smartlist_contains_string(opened_files, file), OP_EQ, 1);
+#ifdef _WIN32
+  // * matches the dot file on Windows
+  tt_int_op(smartlist_contains_string(opened_files, dot), OP_EQ, 1);
+#endif
+
  done:
   SMARTLIST_FOREACH(opened_files, char *, f, tor_free(f));
   smartlist_free(opened_files);
   config_free_lines(result);
   tor_free(torrcd);
   tor_free(subfolder);
-  tor_free(path);
+  tor_free(in_subfolder);
   tor_free(empty);
   tor_free(file);
   tor_free(dot);
@@ -6585,6 +6924,8 @@ struct testcase_t config_tests[] = {
   CONFIG_TEST(include_flag_both_without, TT_FORK),
   CONFIG_TEST(include_flag_torrc_only, TT_FORK),
   CONFIG_TEST(include_flag_defaults_only, TT_FORK),
+  CONFIG_TEST(include_wildcards, 0),
+  CONFIG_TEST(include_hidden, 0),
   CONFIG_TEST(dup_and_filter, 0),
   CONFIG_TEST(check_bridge_distribution_setting_not_a_bridge, TT_FORK),
   CONFIG_TEST(check_bridge_distribution_setting_valid, 0),

src/test/test_helpers.c

@@ -47,6 +47,17 @@
 #include "feature/nodelist/node_st.h"
 #include "feature/nodelist/routerlist_st.h"
 
+#ifdef HAVE_SYS_STAT_H
+#include <sys/stat.h>
+#endif
+
+#ifdef _WIN32
+/* For mkdir() */
+#include <direct.h>
+#else
+#include <dirent.h>
+#endif /* defined(_WIN32) */
+
 #include "test/test.h"
 #include "test/test_helpers.h"
 #include "test/test_connection.h"
@@ -194,6 +205,78 @@ mock_tor_addr_lookup__fail_on_bad_addrs(const char *name,
   return tor_addr_lookup__real(name, family, out);
 }
 
+static char *
+create_directory(const char *parent_dir, const char *name)
+{
+  char *dir = NULL;
+  tor_asprintf(&dir, "%s"PATH_SEPARATOR"%s", parent_dir, name);
+#ifdef _WIN32
+  tt_int_op(mkdir(dir), OP_EQ, 0);
+#else
+  tt_int_op(mkdir(dir, 0700), OP_EQ, 0);
+#endif
+  return dir;
+
+ done:
+  tor_free(dir);
+  return NULL;
+}
+
+static char *
+create_file(const char *parent_dir, const char *name, const char *contents)
+{
+  char *path = NULL;
+  tor_asprintf(&path, "%s"PATH_SEPARATOR"%s", parent_dir, name);
+  contents = contents == NULL ? "" : contents;
+  tt_int_op(write_str_to_file(path, contents, 0), OP_EQ, 0);
+  return path;
+
+ done:
+  tor_free(path);
+  return NULL;
+}
+
+int
+create_test_directory_structure(const char *parent_dir)
+{
+  int ret = -1;
+  char *dir1 = NULL;
+  char *dir2 = NULL;
+  char *file1 = NULL;
+  char *file2 = NULL;
+  char *dot = NULL;
+  char *empty = NULL;
+  char *forbidden = NULL;
+
+  dir1 = create_directory(parent_dir, "dir1");
+  tt_assert(dir1);
+  dir2 = create_directory(parent_dir, "dir2");
+  tt_assert(dir2);
+  file1 = create_file(parent_dir, "file1", "Test 1");
+  tt_assert(file1);
+  file2 = create_file(parent_dir, "file2", "Test 2");
+  tt_assert(file2);
+  dot = create_file(parent_dir, ".test-hidden", "Test .");
+  tt_assert(dot);
+  empty = create_file(parent_dir, "empty", NULL);
+  tt_assert(empty);
+  forbidden = create_directory(parent_dir, "forbidden");
+  tt_assert(forbidden);
+#ifndef _WIN32
+  tt_int_op(chmod(forbidden, 0), OP_EQ, 0);
+#endif
+  ret = 0;
+ done:
+  tor_free(dir1);
+  tor_free(dir2);
+  tor_free(file1);
+  tor_free(file2);
+  tor_free(dot);
+  tor_free(empty);
+  tor_free(forbidden);
+  return ret;
+}
+
 /*********** Helper funcs for making new connections/streams *****************/
 
 /* Helper for test_conn_get_connection() */

src/test/test_helpers.h

@@ -33,6 +33,8 @@ connection_t *test_conn_get_connection(uint8_t state,
                                        uint8_t type, uint8_t purpose);
 or_options_t *helper_parse_options(const char *conf);
 
+int create_test_directory_structure(const char *parent_dir);
+
 extern const char TEST_DESCRIPTORS[];
 
 void *helper_setup_pubsub(const struct testcase_t *);

src/test/test_include.py

@@ -0,0 +1,196 @@
+# Future imports for Python 2.7, mandatory in 3.0
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import errno
+import logging
+import os
+import random
+import socket
+import subprocess
+import sys
+import time
+import re
+
+CONTROL_SOCK_TIMEOUT = 10.0
+LOG_TIMEOUT = 60.0
+LOG_WAIT = 0.1
+
+def fail(msg):
+    logging.error('FAIL')
+    sys.exit(msg)
+
+def skip(msg):
+    logging.warning('SKIP: {}'.format(msg))
+    sys.exit(77)
+
+def wait_for_log(s):
+    cutoff = time.time() + LOG_TIMEOUT
+    while time.time() < cutoff:
+        l = tor_process.stdout.readline()
+        l = l.decode('utf8', 'backslashreplace')
+        if s in l:
+            logging.info('Tor logged: "{}"'.format(l.strip()))
+            return
+        # readline() returns a blank string when there is no output
+        # avoid busy-waiting
+        if len(l) == 0:
+            logging.debug('Tor has not logged anything, waiting for "{}"'.format(s))
+            time.sleep(LOG_WAIT)
+        else:
+            logging.info('Tor logged: "{}", waiting for "{}"'.format(l.strip(), s))
+    fail('Could not find "{}" in logs after {} seconds'.format(s, LOG_TIMEOUT))
+
+def pick_random_port():
+    port = 0
+    random.seed()
+
+    for i in range(8):
+        port = random.randint(10000, 60000)
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        if s.connect_ex(('127.0.0.1', port)) == 0:
+            s.close()
+        else:
+            break
+
+    if port == 0:
+        fail('Could not find a random free port between 10000 and 60000')
+
+    return port
+
+def check_control_list(control_out_file, expected, value_name):
+    received_count = 0
+    for e in expected:
+        received = control_out_file.readline().strip()
+        received_count += 1
+        parts = re.split('[ =-]', received.strip())
+        if len(parts) != 3 or parts[0] != '250' or parts[1] != value_name or parts[2] != e:
+            fail('Unexpected value in response line "{}". Expected {} for value {}'.format(received, e, value_name))
+        if received.startswith('250 '):
+            break
+
+    if received_count != len(expected):
+        fail('Expected response with {} lines but received {} lines'.format(len(expected), received_count))
+
+
+logging.basicConfig(level=logging.DEBUG,
+                    format='%(asctime)s.%(msecs)03d %(message)s',
+                    datefmt='%Y-%m-%d %H:%M:%S')
+
+if sys.hexversion < 0x02070000:
+    fail("ERROR: unsupported Python version (should be >= 2.7)")
+
+if sys.hexversion > 0x03000000 and sys.hexversion < 0x03010000:
+    fail("ERROR: unsupported Python3 version (should be >= 3.1)")
+
+if 'TOR_SKIP_TEST_INCLUDE' in os.environ:
+    skip('$TOR_SKIP_TEST_INCLUDE is set')
+
+control_port = pick_random_port()
+
+assert control_port != 0
+
+if len(sys.argv) < 4:
+     fail('Usage: %s <path-to-tor> <data-dir> <torrc>' % sys.argv[0])
+
+if not os.path.exists(sys.argv[1]):
+    fail('ERROR: cannot find tor at %s' % sys.argv[1])
+if not os.path.exists(sys.argv[2]):
+    fail('ERROR: cannot find datadir at %s' % sys.argv[2])
+if not os.path.exists(sys.argv[3]):
+    fail('ERROR: cannot find torrcdir at %s' % sys.argv[3])
+
+tor_path = sys.argv[1]
+data_dir = sys.argv[2]
+torrc_dir = sys.argv[3]
+
+empty_torrc_path = os.path.join(data_dir, 'empty_torrc')
+open(empty_torrc_path, 'w').close()
+empty_defaults_torrc_path = os.path.join(data_dir, 'empty_defaults_torrc')
+open(empty_defaults_torrc_path, 'w').close()
+torrc = os.path.join(torrc_dir, 'torrc')
+
+tor_process = subprocess.Popen([tor_path,
+                               '-DataDirectory', data_dir,
+                               '-ControlPort', '127.0.0.1:{}'.format(control_port),
+                               '-Log', 'info stdout',
+                               '-LogTimeGranularity', '1',
+                               '-FetchServerDescriptors', '0',
+                               '-DisableNetwork', '1',
+                               '-f', torrc,
+                               '--defaults-torrc', empty_defaults_torrc_path,
+                               ],
+                               stdout=subprocess.PIPE,
+                               stderr=subprocess.PIPE)
+
+if tor_process == None:
+    fail('ERROR: running tor failed')
+
+wait_for_log('Opened Control listener')
+
+control_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+if control_socket.connect_ex(('127.0.0.1', control_port)):
+    tor_process.terminate()
+    fail('Cannot connect to ControlPort')
+control_socket.settimeout(CONTROL_SOCK_TIMEOUT)
+control_out_file = control_socket.makefile('r')
+
+control_socket.sendall('AUTHENTICATE \r\n'.encode('ascii'))
+res = control_out_file.readline().strip()
+if res != '250 OK':
+    tor_process.terminate()
+    fail('Cannot authenticate. Response was: {}'.format(res))
+
+# test configuration file values and order
+control_socket.sendall('GETCONF NodeFamily\r\n'.encode('ascii'))
+check_control_list(control_out_file, ['1', '2', '3', '4', '5', '6', '4' , '5'], 'NodeFamily')
+
+# test reloading the configuration file with seccomp sandbox enabled
+foo_path = os.path.join(torrc_dir, 'torrc.d', 'foo')
+with open(foo_path, 'a') as foo:
+    foo.write('NodeFamily 7')
+
+control_socket.sendall('SIGNAL RELOAD\r\n'.encode('ascii'))
+wait_for_log('Reloading config and resetting internal state.')
+res = control_out_file.readline().strip()
+if res != '250 OK':
+    tor_process.terminate()
+    fail('Cannot reload configuration. Response was: {}'.format(res))
+
+
+control_socket.sendall('GETCONF NodeFamily\r\n'.encode('ascii'))
+check_control_list(control_out_file, ['1', '2', '3', '4', '5', '6', '7', '4' , '5'], 'NodeFamily')
+
+# test that config-can-saveconf is 0 because we have a %include
+control_socket.sendall('getinfo config-can-saveconf\r\n'.encode('ascii'))
+res = control_out_file.readline().strip()
+if res != '250-config-can-saveconf=0':
+    tor_process.terminate()
+    fail('getinfo config-can-saveconf returned wrong response: {}'.format(res))
+else:
+    res = control_out_file.readline().strip()
+    if res != '250 OK':
+        tor_process.terminate()
+        fail('getinfo failed. Response was: {}'.format(res))
+
+# test that saveconf returns error because we have a %include
+control_socket.sendall('SAVECONF\r\n'.encode('ascii'))
+res = control_out_file.readline().strip()
+if res != '551 Unable to write configuration to disk.':
+    tor_process.terminate()
+    fail('SAVECONF returned wrong response. Response was: {}'.format(res))
+
+control_socket.sendall('SIGNAL HALT\r\n'.encode('ascii'))
+
+wait_for_log('exiting cleanly')
+logging.info('OK')
+
+try:
+    tor_process.terminate()
+except OSError as e:
+    if e.errno == errno.ESRCH: # errno 3: No such process
+        # assume tor has already exited due to SIGNAL HALT
+        logging.warn("Tor has already exited")
+    else:
+        raise

src/test/test_include.sh

@@ -0,0 +1,111 @@
+#!/bin/sh
+
+umask 077
+set -e
+set -x
+
+# emulate realpath(), in case coreutils or equivalent is not installed.
+abspath() {
+    f="$*"
+    if [ -d "$f" ]; then
+        dir="$f"
+        base=""
+    else
+        dir="$(dirname "$f")"
+        base="/$(basename "$f")"
+    fi
+    dir="$(cd "$dir" && pwd)"
+    echo "$dir$base"
+}
+
+UNAME_OS=$(uname -s | cut -d_ -f1)
+if test "$UNAME_OS" = 'CYGWIN' || \
+   test "$UNAME_OS" = 'MSYS' || \
+   test "$UNAME_OS" = 'MINGW' || \
+   test "$UNAME_OS" = 'MINGW32' || \
+   test "$UNAME_OS" = 'MINGW64'; then
+  if test "$APPVEYOR" = 'True'; then
+    echo "This test is disabled on Windows CI, as it requires firewall exemptions. Skipping." >&2
+    exit 77
+  fi
+fi
+
+# find the tor binary
+if [ $# -ge 1 ]; then
+  TOR_BINARY="${1}"
+  shift
+else
+  TOR_BINARY="${TESTING_TOR_BINARY:-./src/app/tor}"
+fi
+
+TOR_BINARY="$(abspath "$TOR_BINARY")"
+
+echo "TOR BINARY IS ${TOR_BINARY}"
+
+if "${TOR_BINARY}" --list-modules | grep -q "relay: no"; then
+  echo "This test requires the relay module. Skipping." >&2
+  exit 77
+fi
+
+tmpdir=
+clean () {
+  if [ -n "$tmpdir" ] && [ -d "$tmpdir" ]; then
+    rm -rf "$tmpdir"
+  fi
+}
+
+trap clean EXIT HUP INT TERM
+
+tmpdir="$(mktemp -d -t tor_include_test.XXXXXX)"
+if [ -z "$tmpdir" ]; then
+  echo >&2 mktemp failed
+  exit 2
+elif [ ! -d "$tmpdir" ]; then
+  echo >&2 mktemp failed to make a directory
+  exit 3
+fi
+
+datadir="$tmpdir/data"
+mkdir "$datadir"
+
+configdir="$tmpdir/config"
+mkdir "$configdir"
+
+# translate paths to windows format
+if test "$UNAME_OS" = 'CYGWIN' || \
+   test "$UNAME_OS" = 'MSYS' || \
+   test "$UNAME_OS" = 'MINGW' || \
+   test "$UNAME_OS" = 'MINGW32' || \
+   test "$UNAME_OS" = 'MINGW64'; then
+    datadir=$(cygpath --windows "$datadir")
+    configdir=$(cygpath --windows "$configdir")
+fi
+
+# create test folder structure in configdir
+torrcd="$configdir/torrc.d"
+mkdir "$torrcd"
+mkdir "$torrcd/folder"
+mkdir "$torrcd/empty_folder"
+echo "NodeFamily 1" > "$torrcd/01_one.conf"
+echo "NodeFamily 2" > "$torrcd/02_two.conf"
+echo "NodeFamily 3" > "$torrcd/aa_three.conf"
+echo "NodeFamily 42" > "$torrcd/.hidden.conf"
+echo "NodeFamily 6" > "$torrcd/foo"
+touch "$torrcd/empty.conf"
+echo "# comment" > "$torrcd/comment.conf"
+echo "NodeFamily 4" > "$torrcd/folder/04_four.conf"
+echo "NodeFamily 5" > "$torrcd/folder/05_five.conf"
+torrc="$configdir/torrc"
+echo "Sandbox 1" > "$torrc"
+echo "
+%include $torrcd/*.conf
+%include $torrcd/f*
+%include $torrcd/*/*
+%include $torrcd/empty_folder
+%include $torrcd/empty.conf
+%include $torrcd/comment.conf
+" >> "$torrc"
+
+"${PYTHON:-python}" "${abs_top_srcdir:-.}/src/test/test_include.py" "${TOR_BINARY}" "$datadir" "$configdir"
+
+exit $?

src/test/test_util.c

@@ -18,6 +18,7 @@
 #include "lib/crypt_ops/crypto_rand.h"
 #include "lib/defs/time.h"
 #include "test/test.h"
+#include "test/test_helpers.h"
 #include "lib/memarea/memarea.h"
 #include "lib/process/waitpid.h"
 #include "lib/process/process_win32.h"
@@ -4183,6 +4184,31 @@ test_util_find_str_at_start_of_line(void *ptr)
   ;
 }
 
+static void
+test_util_tor_strreplacechar(void *ptr)
+{
+  (void)ptr;
+  char empty[] = "";
+  char not_contain[] = "bbb";
+  char contains[] = "bab";
+  char contains_all[] = "aaa";
+
+  tor_strreplacechar(empty, 'a', 'b');
+  tt_str_op(empty, OP_EQ, "");
+
+  tor_strreplacechar(not_contain, 'a', 'b');
+  tt_str_op(not_contain, OP_EQ, "bbb");
+
+  tor_strreplacechar(contains, 'a', 'b');
+  tt_str_op(contains, OP_EQ, "bbb");
+
+  tor_strreplacechar(contains_all, 'a', 'b');
+  tt_str_op(contains_all, OP_EQ, "bbb");
+
+ done:
+  ;
+}
+
 static void
 test_util_string_is_C_identifier(void *ptr)
 {
@@ -4409,6 +4435,413 @@ test_util_listdir(void *ptr)
   }
 }
 
+static void
+test_util_glob(void *ptr)
+{
+  (void)ptr;
+
+  smartlist_t *results = NULL;
+  int r, i;
+  char *dir1 = NULL, *dir2 = NULL, *forbidden = NULL, *dirname = NULL;
+  char *expected = NULL, *pattern = NULL;
+  // used for cleanup
+  char *dir1_forbidden = NULL, *dir2_forbidden = NULL;
+  char *forbidden_forbidden = NULL;
+
+  dirname = tor_strdup(get_fname("test_glob"));
+  tt_ptr_op(dirname, OP_NE, NULL);
+
+#ifdef _WIN32
+  r = mkdir(dirname);
+#else
+  r = mkdir(dirname, 0700);
+#endif
+  if (r) {
+    fprintf(stderr, "Can't create directory %s:", dirname);
+    perror("");
+    exit(1);
+  }
+
+  tt_int_op(0, OP_EQ, create_test_directory_structure(dirname));
+  tor_asprintf(&dir1, "%s"PATH_SEPARATOR"dir1", dirname);
+  tor_asprintf(&dir1_forbidden,
+               "%s"PATH_SEPARATOR"dir1"PATH_SEPARATOR"forbidden", dirname);
+  tt_int_op(0, OP_EQ, create_test_directory_structure(dir1));
+  tor_asprintf(&dir2, "%s"PATH_SEPARATOR"dir2", dirname);
+  tor_asprintf(&dir2_forbidden,
+               "%s"PATH_SEPARATOR"dir2"PATH_SEPARATOR"forbidden", dirname);
+  tt_int_op(0, OP_EQ, create_test_directory_structure(dir2));
+  tor_asprintf(&forbidden, "%s"PATH_SEPARATOR"forbidden", dirname);
+  tor_asprintf(&forbidden_forbidden,
+               "%s"PATH_SEPARATOR"forbidden"PATH_SEPARATOR"forbidden",dirname);
+#ifndef _WIN32
+  chmod(forbidden, 0700);
+#endif
+  tt_int_op(0, OP_EQ, create_test_directory_structure(forbidden));
+#ifndef _WIN32
+  chmod(forbidden, 0);
+#endif
+
+#define TEST(input) \
+  do { \
+    tor_asprintf(&pattern, "%s"PATH_SEPARATOR"%s", dirname, input); \
+    results = tor_glob(pattern); \
+    tor_free(pattern); \
+    tt_assert(results); \
+    smartlist_sort_strings(results); \
+  } while (0);
+
+#define EXPECT(result) \
+  do { \
+    tt_int_op(smartlist_len(results), OP_EQ, \
+                            sizeof(result)/sizeof(*result)); \
+    i = 0; \
+    SMARTLIST_FOREACH_BEGIN(results, const char *, f) { \
+      tor_asprintf(&expected, "%s"PATH_SEPARATOR"%s", dirname, result[i]); \
+      tt_str_op(f, OP_EQ, expected); \
+      i++; \
+      tor_free(expected); \
+    } SMARTLIST_FOREACH_END(f); \
+    SMARTLIST_FOREACH(results, char *, f, tor_free(f)); \
+    smartlist_free(results); \
+  } while (0);
+
+#define EXPECT_EMPTY() \
+  do { \
+    tt_int_op(smartlist_len(results), OP_EQ, 0); \
+    SMARTLIST_FOREACH(results, char *, f, tor_free(f)); \
+    smartlist_free(results); \
+  } while (0);
+
+  // wilcards at beginning
+  const char *results_test1[] = {"dir2", "file2"};
+  TEST("*2");
+  EXPECT(results_test1);
+
+  // wildcards at end
+  const char *results_test2[] = {"dir1", "dir2"};
+  TEST("d*");
+  EXPECT(results_test2);
+
+  // wildcards at beginning and end
+#ifdef _WIN32
+  // dot files are not ignored on Windows
+  const char *results_test3[] = {".test-hidden", "dir1", "dir2", "file1",
+                                 "file2", "forbidden"};
+#else
+  const char *results_test3[] = {"dir1", "dir2", "file1", "file2",
+                                 "forbidden"};
+#endif
+  TEST("*i*");
+  EXPECT(results_test3);
+
+  // wildcards in middle
+  const char *results_test4[] = {"dir1", "dir2"};
+  TEST("d?r*");
+  EXPECT(results_test4);
+
+  // test file that does not exist
+  TEST("not-exist");
+  EXPECT_EMPTY();
+
+  // test wildcard that matches nothing
+  TEST("*not-exist*");
+  EXPECT_EMPTY();
+
+  // test path separator at end - no wildcards
+  const char *results_test7[] = {"dir1"};
+  TEST("dir1");
+  EXPECT(results_test7);
+
+  const char *results_test8[] = {"dir1"};
+  TEST("dir1"PATH_SEPARATOR);
+  EXPECT(results_test8);
+
+  const char *results_test9[] = {"file1"};
+  TEST("file1");
+  EXPECT(results_test9);
+
+#if defined(__APPLE__) || defined(__darwin__) || \
+  defined(__FreeBSD__) || defined(__NetBSD__) || defined(OpenBSD)
+  TEST("file1"PATH_SEPARATOR);
+  EXPECT_EMPTY();
+#else
+  const char *results_test10[] = {"file1"};
+  TEST("file1"PATH_SEPARATOR);
+  EXPECT(results_test10);
+#endif
+
+  // test path separator at end - with wildcards and linux path separator
+  const char *results_test11[] = {"dir1", "dir2", "forbidden"};
+  TEST("*/");
+  EXPECT(results_test11);
+
+#ifdef _WIN32
+  // dot files are not ignored on Windows
+  const char *results_test12[] = {".test-hidden", "dir1", "dir2", "empty",
+                                  "file1", "file2", "forbidden"};
+#else
+  const char *results_test12[] = {"dir1", "dir2", "empty", "file1", "file2",
+                                  "forbidden"};
+#endif
+  TEST("*");
+  EXPECT(results_test12);
+
+  // wildcards on folder and file and linux path separator
+  const char *results_test13[] = {"dir1"PATH_SEPARATOR"dir1",
+                                  "dir1"PATH_SEPARATOR"dir2",
+                                  "dir1"PATH_SEPARATOR"file1",
+                                  "dir1"PATH_SEPARATOR"file2",
+                                  "dir2"PATH_SEPARATOR"dir1",
+                                  "dir2"PATH_SEPARATOR"dir2",
+                                  "dir2"PATH_SEPARATOR"file1",
+                                  "dir2"PATH_SEPARATOR"file2"};
+  TEST("?i*/?i*");
+  EXPECT(results_test13);
+
+  // wildcards on file only
+  const char *results_test14[] = {"dir1"PATH_SEPARATOR"dir1",
+                                  "dir1"PATH_SEPARATOR"dir2",
+                                  "dir1"PATH_SEPARATOR"file1",
+                                  "dir1"PATH_SEPARATOR"file2"};
+  TEST("dir1"PATH_SEPARATOR"?i*");
+  EXPECT(results_test14);
+
+  // wildcards on folder only
+  const char *results_test15[] = {"dir1"PATH_SEPARATOR"file1",
+                                  "dir2"PATH_SEPARATOR"file1"};
+  TEST("?i*"PATH_SEPARATOR"file1");
+  EXPECT(results_test15);
+
+  // wildcards after file name
+  TEST("file1"PATH_SEPARATOR"*");
+  EXPECT_EMPTY();
+
+#ifndef _WIN32
+  // test wildcard escaping
+  TEST("\\*");
+  EXPECT_EMPTY();
+
+  // test forbidden directory
+  tor_asprintf(&pattern, "%s"PATH_SEPARATOR"*"PATH_SEPARATOR"*", dirname);
+  results = tor_glob(pattern);
+  tor_free(pattern);
+  tt_assert(!results);
+#endif
+
+#undef TEST
+#undef EXPECT
+#undef EXPECT_EMPTY
+
+ done:
+#ifndef _WIN32
+  chmod(forbidden, 0700);
+  chmod(dir1_forbidden, 0700);
+  chmod(dir2_forbidden, 0700);
+  chmod(forbidden_forbidden, 0700);
+#endif
+  tor_free(dir1);
+  tor_free(dir2);
+  tor_free(forbidden);
+  tor_free(dirname);
+  tor_free(dir1_forbidden);
+  tor_free(dir2_forbidden);
+  tor_free(forbidden_forbidden);
+  tor_free(expected);
+  tor_free(pattern);
+  if (results) {
+    SMARTLIST_FOREACH(results, char *, f, tor_free(f));
+    smartlist_free(results);
+  }
+}
+
+static void
+test_util_get_glob_opened_files(void *ptr)
+{
+  (void)ptr;
+
+  smartlist_t *results = NULL;
+  int r, i;
+  char *dir1 = NULL, *dir2 = NULL, *forbidden = NULL, *dirname = NULL;
+  char *expected = NULL, *pattern = NULL;
+  // used for cleanup
+  char *dir1_forbidden = NULL, *dir2_forbidden = NULL;
+  char *forbidden_forbidden = NULL;
+
+  dirname = tor_strdup(get_fname("test_get_glob_opened_files"));
+  tt_ptr_op(dirname, OP_NE, NULL);
+
+#ifdef _WIN32
+  r = mkdir(dirname);
+#else
+  r = mkdir(dirname, 0700);
+#endif
+  if (r) {
+    fprintf(stderr, "Can't create directory %s:", dirname);
+    perror("");
+    exit(1);
+  }
+
+  tt_int_op(0, OP_EQ, create_test_directory_structure(dirname));
+  tor_asprintf(&dir1, "%s"PATH_SEPARATOR"dir1", dirname);
+  tor_asprintf(&dir1_forbidden,
+               "%s"PATH_SEPARATOR"dir1"PATH_SEPARATOR"forbidden", dirname);
+  tt_int_op(0, OP_EQ, create_test_directory_structure(dir1));
+  tor_asprintf(&dir2, "%s"PATH_SEPARATOR"dir2", dirname);
+  tor_asprintf(&dir2_forbidden,
+               "%s"PATH_SEPARATOR"dir2"PATH_SEPARATOR"forbidden", dirname);
+  tt_int_op(0, OP_EQ, create_test_directory_structure(dir2));
+  tor_asprintf(&forbidden, "%s"PATH_SEPARATOR"forbidden", dirname);
+  tor_asprintf(&forbidden_forbidden,
+               "%s"PATH_SEPARATOR"forbidden"PATH_SEPARATOR"forbidden",dirname);
+#ifndef _WIN32
+  chmod(forbidden, 0700);
+#endif
+  tt_int_op(0, OP_EQ, create_test_directory_structure(forbidden));
+#ifndef _WIN32
+  chmod(forbidden, 0);
+#endif
+
+#define TEST(input) \
+  do { \
+    if (*input) { \
+      tor_asprintf(&pattern, "%s"PATH_SEPARATOR"%s", dirname, input); \
+    } else { /* do not add path separator if empty string */ \
+      tor_asprintf(&pattern, "%s", dirname); \
+    } \
+    results = get_glob_opened_files(pattern); \
+    tor_free(pattern); \
+    tt_assert(results); \
+    smartlist_sort_strings(results); \
+  } while (0);
+
+#define EXPECT(result) \
+  do { \
+    tt_int_op(smartlist_len(results), OP_EQ, \
+                          sizeof(result)/sizeof(*result)); \
+    i = 0; \
+    SMARTLIST_FOREACH_BEGIN(results, const char *, f) { \
+      if (*result[i]) { \
+        tor_asprintf(&expected, "%s"PATH_SEPARATOR"%s", dirname, result[i]); \
+      } else { /* do not add path separator if empty string */ \
+        tor_asprintf(&expected, "%s", dirname); \
+      } \
+      tt_str_op(f, OP_EQ, expected); \
+      i++; \
+      tor_free(expected); \
+    } SMARTLIST_FOREACH_END(f); \
+    SMARTLIST_FOREACH(results, char *, f, tor_free(f)); \
+    smartlist_free(results); \
+  } while (0);
+
+#define EXPECT_EMPTY() \
+  do { \
+    tt_int_op(smartlist_len(results), OP_EQ, 0); \
+    SMARTLIST_FOREACH(results, char *, f, tor_free(f)); \
+    smartlist_free(results); \
+  } while (0);
+
+  // all files on folder
+  const char *results_test1[] = {""}; // only the folder is read
+  TEST("*");
+  EXPECT(results_test1);
+
+  // same as before but ending in path separator
+  const char *results_test2[] = {""}; // only the folder is read
+  TEST("*"PATH_SEPARATOR);
+  EXPECT(results_test2);
+
+  // wilcards in multiple path components
+#ifndef _WIN32
+  const char *results_test3[] = {"", "dir1", "dir2", "empty", "file1", "file2",
+                                 "forbidden"};
+#else
+  // dot files are not special on windows
+  const char *results_test3[] = {"", ".test-hidden", "dir1", "dir2", "empty",
+                                 "file1", "file2", "forbidden"};
+#endif
+  TEST("*"PATH_SEPARATOR"*");
+  EXPECT(results_test3);
+
+  // same as before but ending in path separator
+#ifndef _WIN32
+  const char *results_test4[] = {"", "dir1", "dir2", "empty", "file1", "file2",
+                                 "forbidden"};
+#else
+  // dot files are not special on windows
+  const char *results_test4[] = {"", ".test-hidden", "dir1", "dir2", "empty",
+                                 "file1", "file2", "forbidden"};
+#endif
+  TEST("*"PATH_SEPARATOR"*"PATH_SEPARATOR);
+  EXPECT(results_test4);
+
+  // no glob - folder
+  TEST("");
+  EXPECT_EMPTY();
+
+  // same as before but ending in path separator
+  TEST(PATH_SEPARATOR);
+  EXPECT_EMPTY();
+
+  // no glob - file
+  TEST("file1");
+  EXPECT_EMPTY();
+
+  // same as before but ending in path separator and linux path separator
+  TEST("file1/");
+  EXPECT_EMPTY();
+
+  // file but with wildcard after
+  const char *results_test9[] = {"file1"};
+  TEST("file1"PATH_SEPARATOR"*");
+  EXPECT(results_test9);
+
+  // dir inside dir and linux path separator
+  TEST("dir1/dir1");
+  EXPECT_EMPTY();
+
+  // same as before but ending in path separator
+  TEST("dir1"PATH_SEPARATOR"dir1"PATH_SEPARATOR);
+  EXPECT_EMPTY();
+
+  // no glob - empty
+  TEST("empty");
+  EXPECT_EMPTY();
+
+  // same as before but ending in path separator
+  TEST("empty"PATH_SEPARATOR);
+  EXPECT_EMPTY();
+
+  // no glob - does not exist
+  TEST("not_exist");
+  EXPECT_EMPTY();
+
+#undef TEST
+#undef EXPECT
+#undef EXPECT_EMPTY
+
+ done:
+#ifndef _WIN32
+  chmod(forbidden, 0700);
+  chmod(dir1_forbidden, 0700);
+  chmod(dir2_forbidden, 0700);
+  chmod(forbidden_forbidden, 0700);
+#endif
+  tor_free(dir1);
+  tor_free(dir2);
+  tor_free(forbidden);
+  tor_free(dirname);
+  tor_free(dir1_forbidden);
+  tor_free(dir2_forbidden);
+  tor_free(forbidden_forbidden);
+  tor_free(expected);
+  tor_free(pattern);
+  if (results) {
+    SMARTLIST_FOREACH(results, char *, f, tor_free(f));
+    smartlist_free(results);
+  }
+}
+
 static void
 test_util_parent_dir(void *ptr)
 {
@@ -6581,10 +7014,13 @@ struct testcase_t util_tests[] = {
   UTIL_TEST(laplace, 0),
   UTIL_TEST(clamp_double_to_int64, 0),
   UTIL_TEST(find_str_at_start_of_line, 0),
+  UTIL_TEST(tor_strreplacechar, 0),
   UTIL_TEST(string_is_C_identifier, 0),
   UTIL_TEST(string_is_utf8, 0),
   UTIL_TEST(asprintf, 0),
   UTIL_TEST(listdir, 0),
+  UTIL_TEST(glob, 0),
+  UTIL_TEST(get_glob_opened_files, 0),
   UTIL_TEST(parent_dir, 0),
   UTIL_TEST(ftruncate, 0),
   UTIL_TEST(nowrap_math, 0),

src/tools/include.am

@@ -11,7 +11,7 @@ src_tools_tor_resolve_LDADD = \
 	$(TOR_UTIL_LIBS) \
         $(TOR_CRYPTO_LIBS) $(TOR_LIBS_CRYPTLIB)\
 	$(rust_ldadd) \
-	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_USERENV@
+	@TOR_LIB_MATH@ @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_USERENV@
 
 if COVERAGE_ENABLED
 src_tools_tor_cov_resolve_SOURCES = src/tools/tor-resolve.c
@@ -36,7 +36,7 @@ src_tools_tor_gencert_LDADD = \
 	$(TOR_UTIL_LIBS) \
 	$(rust_ldadd) \
 	@TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ $(TOR_LIBS_CRYPTLIB) \
-	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@
+	@TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @TOR_LIB_USERENV@ @CURVE25519_LIBS@
 endif
 
 src_tools_tor_print_ed_signing_cert_SOURCES = src/tools/tor-print-ed-signing-cert.c
@@ -46,7 +46,7 @@ src_tools_tor_print_ed_signing_cert_LDADD = \
         $(TOR_CRYPTO_LIBS) \
         $(TOR_UTIL_LIBS) \
 	@TOR_LIB_MATH@ $(TOR_LIBS_CRYPTLIB) \
-	@TOR_LIB_WS32@ @TOR_LIB_USERENV@ @TOR_LIB_GDI@
+	@TOR_LIB_WS32@ @TOR_LIB_USERENV@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@
 
 if USE_NSS
 # ...
@@ -61,7 +61,7 @@ src_tools_tor_cov_gencert_LDADD = \
     $(TOR_CRYPTO_TESTING_LIBS) \
     $(TOR_UTIL_TESTING_LIBS) \
     @TOR_LIB_MATH@ @TOR_ZLIB_LIBS@ $(TOR_LIBS_CRYPTLIB) \
-    @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
+    @TOR_LIB_WS32@ @TOR_LIB_IPHLPAPI@ @TOR_LIB_SHLWAPI@ @TOR_LIB_GDI@ @CURVE25519_LIBS@
 endif
 endif