From 2a3eef4404415f1bb90a86c9cc396b6dee89039c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:05:01 +0200 Subject: [PATCH 01/34] Remove unused `int pid` member of `managed_proxy_t`. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.h | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index d304dcd485..b80875c95c 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -97,8 +97,6 @@ typedef struct { /* A pointer to the process handle of this managed proxy. */ struct process_handle_t *process_handle; - int pid; /* The Process ID this managed proxy is using. */ - /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport * plugins that use it. */ From 5f26ae833eea79e56cb8cb8133b16a9cb0944ae4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:23:59 +0200 Subject: [PATCH 02/34] Refactor read_to_chunk() such that it supports both pipes and sockets. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index c52ea2784e..bd420510ab 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -21,6 +21,7 @@ #endif #include +#include #ifdef PARANOIA /** Helper: If PARANOIA is defined, assert that the buffer in local variable @@ -30,27 +31,33 @@ #define check() STMT_NIL #endif /* defined(PARANOIA) */ -/** Read up to at_most bytes from the socket fd into +/** Read up to at_most bytes from the file descriptor fd into * chunk (which must be on buf). If we get an EOF, set * *reached_eof to 1. Return -1 on error, 0 on eof or blocking, * and the number of bytes read otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, - int *reached_eof, int *socket_error) + int *reached_eof, int *error, bool is_socket) { ssize_t read_result; if (at_most > CHUNK_REMAINING_CAPACITY(chunk)) at_most = CHUNK_REMAINING_CAPACITY(chunk); - read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + + if (is_socket) + read_result = tor_socket_recv(fd, CHUNK_WRITE_PTR(chunk), at_most, 0); + else + read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { int e = tor_socket_errno(fd); if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) - log_warn(LD_NET,"recv() failed: WSAENOBUFS. Not enough ram?"); + log_warn(LD_NET, "%s() failed: WSAENOBUFS. Not enough ram?", + is_socket ? "recv" : "read"); #endif - *socket_error = e; + if (error) + *error = e; return -1; } return 0; /* would block. */ @@ -108,7 +115,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error); + r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error, true); check(); if (r < 0) return r; /* Error */ From 340260281a514ee92b4c40ee5ddf8728ac580a40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 13:28:01 +0200 Subject: [PATCH 03/34] Refactor flush_chunk() to work on pipes as well as sockets. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index bd420510ab..fc133a01fe 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -129,22 +129,26 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, } /** Helper for buf_flush_to_socket(): try to write sz bytes from chunk - * chunk of buffer buf onto socket s. On success, deduct - * the bytes written from *buf_flushlen. Return the number of bytes - * written on success, 0 on blocking, -1 on failure. + * chunk of buffer buf onto file descriptor fd. On + * success, deduct the bytes written from *buf_flushlen. Return the + * number of bytes written on success, 0 on blocking, -1 on failure. */ static inline int -flush_chunk(tor_socket_t s, buf_t *buf, chunk_t *chunk, size_t sz, - size_t *buf_flushlen) +flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, + size_t *buf_flushlen, bool is_socket) { ssize_t write_result; if (sz > chunk->datalen) sz = chunk->datalen; - write_result = tor_socket_send(s, chunk->data, sz, 0); + + if (is_socket) + write_result = tor_socket_send(fd, chunk->data, sz, 0); + else + write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(s); + int e = tor_socket_errno(fd); if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -195,7 +199,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen); + r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen, true); check(); if (r < 0) return r; From c71f9df07bb5f9f01a47a66f245c7794cb2bd839 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:29:27 +0200 Subject: [PATCH 04/34] Refactor buf_flush_to_socket() into buf_flush_to_fd(). This patch refactors buf_flush_to_socket() into buf_flush_to_fd() and creates a specialization function for buf_flush_to_socket() that makes use of buf_flush_to_fd(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index fc133a01fe..2ffca8b7da 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -166,15 +166,15 @@ flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, } } -/** Write data from buf to the socket s. Write at most +/** Write data from buf to the file descriptor fd. Write at most * sz bytes, decrement *buf_flushlen by * the number of bytes actually written, and remove the written bytes * from the buffer. Return the number of bytes written on success, * -1 on failure. Return 0 if write() would block. */ -int -buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, - size_t *buf_flushlen) +static int +buf_flush_to_fd(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen, bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes flushed" are not mutually exclusive. @@ -182,7 +182,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, int r; size_t flushed = 0; tor_assert(buf_flushlen); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(*buf_flushlen > buf->datalen)) { *buf_flushlen = buf->datalen; } @@ -199,7 +199,7 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, else flushlen0 = buf->head->datalen; - r = flush_chunk(s, buf, buf->head, flushlen0, buf_flushlen, true); + r = flush_chunk(fd, buf, buf->head, flushlen0, buf_flushlen, is_socket); check(); if (r < 0) return r; @@ -211,3 +211,16 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, tor_assert(flushed < INT_MAX); return (int)flushed; } + +/** Write data from buf to the socket s. Write at most + * sz bytes, decrement *buf_flushlen by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); +} From 771930b84cb4aeeb75e175368108697bafd94a2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:35:38 +0200 Subject: [PATCH 05/34] Refactor buf_read_from_socket() into buf_read_from_fd(). This patch refactors buf_read_from_socket() into buf_read_from_fd(), and creates a specialized function for buf_read_from_socket(), which uses buf_read_from_fd(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 35 +++++++++++++++++++++++++---------- 1 file changed, 25 insertions(+), 10 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 2ffca8b7da..7c81096e38 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -75,16 +75,17 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, } } -/** Read from socket s, writing onto end of buf. Read at most - * at_most bytes, growing the buffer as necessary. If recv() returns 0 - * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on - * error; else return the number of bytes read. +/** Read from file descriptor fd, writing onto end of buf. Read + * at most at_most bytes, growing the buffer as necessary. If recv() + * returns 0 (because of EOF), set *reached_eof to 1 and return 0. + * Return -1 on error; else return the number of bytes read. */ /* XXXX indicate "read blocked" somehow? */ -int -buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, - int *reached_eof, - int *socket_error) +static int +buf_read_from_fd(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error, + bool is_socket) { /* XXXX It's stupid to overload the return values for these functions: * "error status" and "number of bytes read" are not mutually exclusive. @@ -94,7 +95,7 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, check(); tor_assert(reached_eof); - tor_assert(SOCKET_OK(s)); + tor_assert(SOCKET_OK(fd)); if (BUG(buf->datalen >= INT_MAX)) return -1; @@ -115,7 +116,8 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, readlen = cap; } - r = read_to_chunk(buf, chunk, s, readlen, reached_eof, socket_error, true); + r = read_to_chunk(buf, chunk, fd, readlen, + reached_eof, socket_error, is_socket); check(); if (r < 0) return r; /* Error */ @@ -224,3 +226,16 @@ buf_flush_to_socket(buf_t *buf, tor_socket_t s, size_t sz, { return buf_flush_to_fd(buf, s, sz, buf_flushlen, true); } + +/** Read from socket s, writing onto end of buf. Read at most + * at_most bytes, growing the buffer as necessary. If recv() returns 0 + * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); +} From 31b3a6577c89492e94836f6e3b4bfc7051a3dc7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 14:42:29 +0200 Subject: [PATCH 06/34] Add buf_flush_to_pipe() and buf_read_from_pipe(). This patch adds two new functions: buf_flush_to_pipe() and buf_read_from_pipe(), which makes use of our new buf_flush_to_fd() and buf_read_from_fd() functions. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 26 ++++++++++++++++++++++++++ src/lib/net/buffers_net.h | 7 +++++++ 2 files changed, 33 insertions(+) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 7c81096e38..1b65819dbb 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -239,3 +239,29 @@ buf_read_from_socket(buf_t *buf, tor_socket_t s, size_t at_most, { return buf_read_from_fd(buf, s, at_most, reached_eof, socket_error, true); } + +/** Write data from buf to the pipe fd. Write at most + * sz bytes, decrement *buf_flushlen by + * the number of bytes actually written, and remove the written bytes + * from the buffer. Return the number of bytes written on success, + * -1 on failure. Return 0 if write() would block. + */ +int +buf_flush_to_pipe(buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen) +{ + return buf_flush_to_fd(buf, fd, sz, buf_flushlen, false); +} + +/** Read from pipe fd, writing onto end of buf. Read at most + * at_most bytes, growing the buffer as necessary. If read() returns 0 + * (because of EOF), set *reached_eof to 1 and return 0. Return -1 on + * error; else return the number of bytes read. + */ +int +buf_read_from_pipe(buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error) +{ + return buf_read_from_fd(buf, fd, at_most, reached_eof, socket_error, false); +} diff --git a/src/lib/net/buffers_net.h b/src/lib/net/buffers_net.h index 417f6f9413..8911b082a2 100644 --- a/src/lib/net/buffers_net.h +++ b/src/lib/net/buffers_net.h @@ -24,4 +24,11 @@ int buf_read_from_socket(struct buf_t *buf, tor_socket_t s, size_t at_most, int buf_flush_to_socket(struct buf_t *buf, tor_socket_t s, size_t sz, size_t *buf_flushlen); +int buf_read_from_pipe(struct buf_t *buf, int fd, size_t at_most, + int *reached_eof, + int *socket_error); + +int buf_flush_to_pipe(struct buf_t *buf, int fd, size_t sz, + size_t *buf_flushlen); + #endif /* !defined(TOR_BUFFERS_H) */ From 2b41b857bdabffeafbbf748189b22cd6ee818394 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 10 Sep 2018 15:15:45 +0200 Subject: [PATCH 07/34] Add LD_PROCESS as log domain. See: https://bugs.torproject.org/28179 --- doc/tor.1.txt | 3 ++- src/lib/log/log.c | 2 +- src/lib/log/log.h | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index e94382b66b..42fe8f1bcc 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -669,7 +669,8 @@ GENERAL OPTIONS + The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, - acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, and dos. + acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, and + process. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 46107fe848..861de6e963 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1268,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", NULL }; /** Return a bitmask for the log domain for which domain is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 493409756f..1cd6087eb8 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -107,8 +107,10 @@ #define LD_CONSDIFF (1u<<24) /** Denial of Service mitigation. */ #define LD_DOS (1u<<25) +/** Processes */ +#define LD_PROCESS (1u<<26) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 26 +#define N_LOGGING_DOMAINS 27 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ From 35509978dd4985901431abe895d1443e75afc00a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:25:11 +0100 Subject: [PATCH 08/34] Add new Process subsystem. This patch adds a new Process subsystem for running external programs in the background of Tor. The design is focused around a new type named `process_t` which have an API that allows the developer to easily write code that interacts with the given child process. These interactions includes: - Easy API for writing output to the child process's standard input handle. - Receive callbacks whenever the child has output on either its standard output or standard error handles. - Receive callback when the child process terminates. We also support two different "protocols" for handling output from the child process. The default protocol is the "line" protocol where the process output callbacks will be invoked only when there is complete lines (either "\r\n" or "\n" terminated). We also support the "raw" protocol where the read callbacks will get whatever the operating system delivered to us in a single read operation. This patch does not include any operating system backends, but the Unix and Windows backends will be included in separate commits. See: https://bugs.torproject.org/28179 --- src/app/main/main.c | 6 + src/lib/process/.may_include | 5 +- src/lib/process/include.am | 2 + src/lib/process/process.c | 644 +++++++++++++++++++++++++++++++++++ src/lib/process/process.h | 127 +++++++ src/test/include.am | 1 + src/test/test.c | 1 + src/test/test.h | 1 + src/test/test_process.c | 558 ++++++++++++++++++++++++++++++ 9 files changed, 1343 insertions(+), 2 deletions(-) create mode 100644 src/lib/process/process.c create mode 100644 src/lib/process/process.h create mode 100644 src/test/test_process.c diff --git a/src/app/main/main.c b/src/app/main/main.c index 03b3a95d03..15b6b48ff2 100644 --- a/src/app/main/main.c +++ b/src/app/main/main.c @@ -73,6 +73,7 @@ #include "lib/geoip/geoip.h" #include "lib/process/waitpid.h" +#include "lib/process/process.h" #include "lib/meminfo/meminfo.h" #include "lib/osinfo/uname.h" @@ -557,6 +558,10 @@ tor_init(int argc, char *argv[]) rend_cache_init(); addressmap_init(); /* Init the client dns cache. Do it always, since it's * cheap. */ + + /* Initialize Process subsystem. */ + process_init(); + /* Initialize the HS subsystem. */ hs_init(); @@ -784,6 +789,7 @@ tor_free_all(int postfork) circuitmux_ewma_free_all(); accounting_free_all(); protover_summary_cache_free_all(); + process_free_all(); if (!postfork) { config_free_all(); diff --git a/src/lib/process/.may_include b/src/lib/process/.may_include index a2d57a52f3..b1c50c24a6 100644 --- a/src/lib/process/.may_include +++ b/src/lib/process/.may_include @@ -4,8 +4,9 @@ lib/cc/*.h lib/container/*.h lib/ctime/*.h lib/err/*.h -lib/intmath/*.h +lib/evloop/*.h lib/fs/*.h +lib/intmath/*.h lib/log/*.h lib/malloc/*.h lib/net/*.h @@ -15,4 +16,4 @@ lib/subsys/*.h lib/testsupport/*.h lib/thread/*.h -ht.h \ No newline at end of file +ht.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index 2aa30cc3c1..1d213d1c00 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -9,6 +9,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/daemon.c \ src/lib/process/env.c \ src/lib/process/pidfile.c \ + src/lib/process/process.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -24,6 +25,7 @@ noinst_HEADERS += \ src/lib/process/daemon.h \ src/lib/process/env.h \ src/lib/process/pidfile.h \ + src/lib/process/process.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c new file mode 100644 index 0000000000..d3967a52da --- /dev/null +++ b/src/lib/process/process.c @@ -0,0 +1,644 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.c + * \brief Module for working with other processes. + **/ + +#define PROCESS_PRIVATE +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/env.h" + +#ifdef HAVE_STDDEF_H +#include +#endif + +/** A list of all process_t instances currently allocated. */ +static smartlist_t *processes; + +/** Structure to represent a child process. */ +struct process_t { + /** Process status. */ + process_status_t status; + + /** Which protocol is the process using? */ + process_protocol_t protocol; + + /** Which function to call when we have data ready from stdout? */ + process_read_callback_t stdout_read_callback; + + /** Which function to call when we have data ready from stderr? */ + process_read_callback_t stderr_read_callback; + + /** Which function call when our process terminated? */ + process_exit_callback_t exit_callback; + + /** Our exit code when the process have terminated. */ + process_exit_code_t exit_code; + + /** Name of the command we want to execute (for example: /bin/ls). */ + char *command; + + /** The arguments used for the new process. */ + smartlist_t *arguments; + + /** The environment used for the new process. */ + smartlist_t *environment; + + /** Buffer to store data from stdout when it is read. */ + buf_t *stdout_buffer; + + /** Buffer to store data from stderr when it is read. */ + buf_t *stderr_buffer; + + /** Buffer to store data to stdin before it is written. */ + buf_t *stdin_buffer; + + /** Do we need to store some custom data with the process? */ + void *data; +}; + +/** Convert a given process status in status to its string + * representation. */ +const char * +process_status_to_string(process_status_t status) +{ + switch (status) { + case PROCESS_STATUS_NOT_RUNNING: + return "not running"; + case PROCESS_STATUS_RUNNING: + return "running"; + case PROCESS_STATUS_ERROR: + return "error"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Convert a given process protocol in protocol to its string + * representation. */ +const char * +process_protocol_to_string(process_protocol_t protocol) +{ + switch (protocol) { + case PROCESS_PROTOCOL_LINE: + return "Line"; + case PROCESS_PROTOCOL_RAW: + return "Raw"; + } + + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return NULL; + /* LCOV_EXCL_STOP */ +} + +/** Initialize the Process subsystem. This function initializes the Process + * subsystem's global state. For cleaning up, process_free_all() should + * be called. */ +void +process_init(void) +{ + processes = smartlist_new(); +} + +/** Free up all resources that is handled by the Process subsystem. Note that + * this call does not terminate already running processes. */ +void +process_free_all(void) +{ + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); + smartlist_free(processes); +} + +/** Get a list of all processes. This function returns a smartlist of + * process_t containing all the currently allocated processes. */ +const smartlist_t * +process_get_all_processes(void) +{ + return processes; +} + +/** Allocate and initialize a new process. This function returns a newly + * allocated and initialized process data, which can be used to configure and + * later run a subprocess of Tor. Use the various process_set_*() + * methods to configure it and run the process using process_exec(). Use + * command to specify the path to the command to run. You can either + * specify an absolute path to the command or relative where Tor will use the + * underlying operating system's functionality for finding the command to run. + * */ +process_t * +process_new(const char *command) +{ + tor_assert(command); + + process_t *process; + process = tor_malloc_zero(sizeof(process_t)); + + /* Set our command. */ + process->command = tor_strdup(command); + + /* By default we are not running. */ + process->status = PROCESS_STATUS_NOT_RUNNING; + + /* Prepare process environment. */ + process->arguments = smartlist_new(); + process->environment = smartlist_new(); + + /* Prepare the buffers. */ + process->stdout_buffer = buf_new(); + process->stderr_buffer = buf_new(); + process->stdin_buffer = buf_new(); + + smartlist_add(processes, process); + + return process; +} + +/** Deallocate the given process in process. */ +void +process_free_(process_t *process) +{ + if (! process) + return; + + /* Cleanup parameters. */ + tor_free(process->command); + + /* Cleanup arguments and environment. */ + SMARTLIST_FOREACH(process->arguments, char *, x, tor_free(x)); + smartlist_free(process->arguments); + + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + + /* Cleanup the buffers. */ + buf_free(process->stdout_buffer); + buf_free(process->stderr_buffer); + buf_free(process->stdin_buffer); + + smartlist_remove(processes, process); + + tor_free(process); +} + +/** Execute the given process. This function executes the given process as a + * subprocess of Tor. Returns PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_exec(process_t *process) +{ + tor_assert(process); + + process_status_t status = PROCESS_STATUS_NOT_RUNNING; + + log_info(LD_PROCESS, "Starting new process: %s", process->command); + + /* Update our state. */ + process_set_status(process, status); + + if (status != PROCESS_STATUS_RUNNING) { + log_warn(LD_PROCESS, "Failed to start process: %s", + process_get_command(process)); + } + + return status; +} + +/** Set the callback function for output from the child process's standard out + * handle. This function sets the callback function which is called every time + * the child process have written output to its standard out file handle. + * + * Use process_set_protocol(process, PROCESS_PROTOCOL_LINE) if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stdout_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stdout_read_callback = callback; +} + +/** Set the callback function for output from the child process's standard + * error handle. This function sets the callback function which is called + * every time the child process have written output to its standard error file + * handle. + * + * Use process_set_protocol(process, PROCESS_PROTOCOL_LINE) if you want + * the callback to only contain complete "\n" or "\r\n" terminated lines. */ +void +process_set_stderr_read_callback(process_t *process, + process_read_callback_t callback) +{ + tor_assert(process); + process->stderr_read_callback = callback; +} + +/** Set the callback function for process exit notification. The + * callback function will be called every time your child process have + * terminated. */ +void +process_set_exit_callback(process_t *process, + process_exit_callback_t callback) +{ + tor_assert(process); + process->exit_callback = callback; +} + +/** Get the current command of the given process. */ +const char * +process_get_command(const process_t *process) +{ + tor_assert(process); + return process->command; +} + +void +process_set_protocol(process_t *process, process_protocol_t protocol) +{ + tor_assert(process); + process->protocol = protocol; +} + +/** Get the currently used protocol of the given process. */ +process_protocol_t +process_get_protocol(const process_t *process) +{ + tor_assert(process); + return process->protocol; +} + +/** Set opague pointer to data. This function allows you to store a pointer to + * your own data in the given process. Use process_get_data() in the + * various callback functions to retrieve the data again. + * + * Note that the given process does NOT take ownership of the data and you are + * responsible for freeing up any resources allocated by the given data. + * */ +void +process_set_data(process_t *process, void *data) +{ + tor_assert(process); + process->data = data; +} + +/** Get the opaque pointer to callback data from the given process. This + * function allows you get the data you stored with process_set_data() + * in the different callback functions. */ +void * +process_get_data(const process_t *process) +{ + tor_assert(process); + return process->data; +} + +/** Set the status of a given process. */ +void +process_set_status(process_t *process, process_status_t status) +{ + tor_assert(process); + process->status = status; +} + +/** Get the status of the given process. */ +process_status_t +process_get_status(const process_t *process) +{ + tor_assert(process); + return process->status; +} + +/** Append an argument to the list of arguments in the given process. */ +void +process_append_argument(process_t *process, const char *argument) +{ + tor_assert(process); + tor_assert(argument); + + smartlist_add(process->arguments, tor_strdup(argument)); +} + +/** Returns a list of arguments (excluding the command itself) from the + * given process. */ +const smartlist_t * +process_get_arguments(const process_t *process) +{ + tor_assert(process); + return process->arguments; +} + +/** Returns a newly allocated Unix style argument vector. Use tor_free() + * to deallocate it after use. */ +char ** +process_get_argv(const process_t *process) +{ + tor_assert(process); + + /** Generate a Unix style process argument vector from our process's + * arguments smartlist_t. */ + char **argv = NULL; + + char *filename = process->command; + const smartlist_t *arguments = process->arguments; + const size_t size = smartlist_len(arguments); + + /* Make space for the process filename as argv[0] and a trailing NULL. */ + argv = tor_malloc_zero(sizeof(char *) * (size + 2)); + + /* Set our filename as first argument. */ + argv[0] = filename; + + /* Put in the rest of the values from arguments. */ + SMARTLIST_FOREACH_BEGIN(arguments, char *, arg_val) { + tor_assert(arg_val != NULL); + + argv[arg_val_sl_idx + 1] = arg_val; + } SMARTLIST_FOREACH_END(arg_val); + + return argv; +} + +/** Set the given key/value pair as environment variable in the + * given process. */ +void +process_set_environment(process_t *process, + const char *key, + const char *value) +{ + tor_assert(process); + tor_assert(key); + tor_assert(value); + + smartlist_add_asprintf(process->environment, "%s=%s", key, value); +} + +/** Returns a newly allocated process_environment_t containing the + * environment variables for the given process. */ +process_environment_t * +process_get_environment(const process_t *process) +{ + tor_assert(process); + return process_environment_make(process->environment); +} + +/** Write size bytes of data to the given process's standard + * input. */ +void +process_write(process_t *process, + const uint8_t *data, size_t size) +{ + tor_assert(process); + tor_assert(data); + + buf_add(process->stdin_buffer, (char *)data, size); + process_write_stdin(process, process->stdin_buffer); +} + +/** As tor_vsnprintf(), but write the data to the given process's standard + * input. */ +void +process_vprintf(process_t *process, + const char *format, va_list args) +{ + tor_assert(process); + tor_assert(format); + + int size; + char *data; + + size = tor_vasprintf(&data, format, args); + process_write(process, (uint8_t *)data, size); + tor_free(data); +} + +/** As tor_snprintf(), but write the data to the given process's standard + * input. */ +void +process_printf(process_t *process, + const char *format, ...) +{ + tor_assert(process); + tor_assert(format); + + va_list ap; + va_start(ap, format); + process_vprintf(process, format, ap); + va_end(ap); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard output + * handle. */ +void +process_notify_event_stdout(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stdout(process, process->stdout_buffer); + + if (ret > 0) + process_read_data(process, + process->stdout_buffer, + process->stdout_read_callback); +} + +/** This function is called by the Process backend when a given process have + * data that is ready to be read from the child process's standard error + * handle. */ +void +process_notify_event_stderr(process_t *process) +{ + tor_assert(process); + + int ret; + ret = process_read_stderr(process, process->stderr_buffer); + + if (ret > 0) + process_read_data(process, + process->stderr_buffer, + process->stderr_read_callback); +} + +/** This function is called by the Process backend when a given process is + * allowed to begin writing data to the standard input of the child process. */ +void +process_notify_event_stdin(process_t *process) +{ + tor_assert(process); + + process_write_stdin(process, process->stdin_buffer); +} + +/** This function is called by the Process backend when a given process have + * terminated. The exit status code is passed in exit_code. We mark the + * process as no longer running and calls the exit_callback with + * information about the process termination. */ +void +process_notify_event_exit(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_debug(LD_PROCESS, + "Process terminated with exit code: %"PRIu64, exit_code); + + /* Update our state. */ + process_set_status(process, PROCESS_STATUS_NOT_RUNNING); + process->exit_code = exit_code; + + /* Call our exit callback, if it exists. */ + if (process->exit_callback) { + process->exit_callback(process, exit_code); + } +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard out handle. Returns the number of + * bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + + return 0; +} + +/** This function is called whenever the Process backend have notified us that + * there is data to be read from its standard error handle. Returns the number + * of bytes that have been put into the given buffer. */ +MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); + + return 0; +} + +/** This function calls the backend function for the given process whenever + * there is data to be written to the backends' file handles. */ +MOCK_IMPL(STATIC void, process_write_stdin, + (process_t *process, buf_t *buffer)) +{ + tor_assert(process); + tor_assert(buffer); +} + +/** This function calls the protocol handlers based on the value of + * process_get_protocol(process). Currently we call + * process_read_buffer() for PROCESS_PROTOCOL_RAW and + * process_read_lines() for PROCESS_PROTOCOL_LINE. */ +STATIC void +process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + switch (process_get_protocol(process)) { + case PROCESS_PROTOCOL_RAW: + process_read_buffer(process, buffer, callback); + break; + case PROCESS_PROTOCOL_LINE: + process_read_lines(process, buffer, callback); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_unreached(); + return; + /* LCOV_EXCL_STOP */ + } +} + +/** This function takes the content of the given buffer and passes it to + * the given callback function, but ensures that an additional zero byte + * is added to the end of the data such that the given callback implementation + * can threat the content as a ASCIIZ string. */ +STATIC void +process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer); + + /* We allocate an extra byte for the zero byte in the end. */ + char *data = tor_malloc_zero(size + 1); + + buf_get_bytes(buffer, data, size); + log_debug(LD_PROCESS, "Read data from process"); + + if (callback) + callback(process, data, size); + + tor_free(data); +} + +/** This function tries to extract complete lines from the given buffer + * and calls the given callback function whenever it has a complete + * line. Before calling callback we remove the trailing "\n" or "\r\n" + * from the line. If we are unable to extract a complete line we leave the data + * in the buffer for next call. */ +STATIC void +process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback) +{ + tor_assert(process); + tor_assert(buffer); + + const size_t size = buf_datalen(buffer) + 1; + size_t line_size = 0; + char *data = tor_malloc_zero(size); + int ret; + + while (true) { + line_size = size; + ret = buf_get_line(buffer, data, &line_size); + + /* A complete line should always be smaller than the size of our + * buffer. */ + tor_assert(ret != -1); + + /* Remove \n from the end of the line. */ + if (data[line_size - 1] == '\n') { + data[line_size - 1] = '\0'; + --line_size; + } + + /* Remove \r from the end of the line. */ + if (data[line_size - 1] == '\r') { + data[line_size - 1] = '\0'; + --line_size; + } + + if (ret == 1) { + log_debug(LD_PROCESS, "Read line from process: \"%s\"", data); + + if (callback) + callback(process, data, line_size); + + /* We have read a whole line, let's see if there is more lines to read. + * */ + continue; + } + + /* No complete line for us to read. We are done for now. */ + tor_assert_nonfatal(ret == 0); + break; + } + + tor_free(data); +} diff --git a/src/lib/process/process.h b/src/lib/process/process.h new file mode 100644 index 0000000000..cf20a5d80b --- /dev/null +++ b/src/lib/process/process.h @@ -0,0 +1,127 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process.h + * \brief Header for process.c + **/ + +#ifndef TOR_PROCESS_H +#define TOR_PROCESS_H + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/string/printf.h" + +/** Maximum number of bytes to write to a process' stdin. */ +#define PROCESS_MAX_WRITE (1024) + +/** Maximum number of bytes to read from a process' stdout/stderr. */ +#define PROCESS_MAX_READ (1024) + +typedef enum { + /** The process is not running. */ + PROCESS_STATUS_NOT_RUNNING, + + /** The process is running. */ + PROCESS_STATUS_RUNNING, + + /** The process is in an erroneous state. */ + PROCESS_STATUS_ERROR +} process_status_t; + +const char *process_status_to_string(process_status_t status); + +typedef enum { + /** Pass complete \n-terminated lines to the + * callback (with the \n or \r\n removed). */ + PROCESS_PROTOCOL_LINE, + + /** Pass the raw response from read() to the callback. */ + PROCESS_PROTOCOL_RAW +} process_protocol_t; + +const char *process_protocol_to_string(process_protocol_t protocol); + +struct process_t; +typedef struct process_t process_t; + +typedef uint64_t process_exit_code_t; + +typedef void (*process_read_callback_t)(process_t *, + char *, + size_t); +typedef void (*process_exit_callback_t)(process_t *, + process_exit_code_t); + +void process_init(void); +void process_free_all(void); +const smartlist_t *process_get_all_processes(void); + +process_t *process_new(const char *command); +void process_free_(process_t *process); +#define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) + +process_status_t process_exec(process_t *process); + +void process_set_stdout_read_callback(process_t *, + process_read_callback_t); +void process_set_stderr_read_callback(process_t *, + process_read_callback_t); +void process_set_exit_callback(process_t *, + process_exit_callback_t); + +const char *process_get_command(const process_t *process); + +void process_append_argument(process_t *process, const char *argument); +const smartlist_t *process_get_arguments(const process_t *process); +char **process_get_argv(const process_t *process); + +void process_set_environment(process_t *process, + const char *key, + const char *value); + +struct process_environment_t; +struct process_environment_t *process_get_environment(const process_t *); + +void process_set_protocol(process_t *process, process_protocol_t protocol); +process_protocol_t process_get_protocol(const process_t *process); + +void process_set_data(process_t *process, void *data); +void *process_get_data(const process_t *process); + +void process_set_status(process_t *process, process_status_t status); +process_status_t process_get_status(const process_t *process); + +void process_write(process_t *process, + const uint8_t *data, size_t size); +void process_vprintf(process_t *process, + const char *format, va_list args) CHECK_PRINTF(2, 0); +void process_printf(process_t *process, + const char *format, ...) CHECK_PRINTF(2, 3); + +void process_notify_event_stdout(process_t *process); +void process_notify_event_stderr(process_t *process); +void process_notify_event_stdin(process_t *process); +void process_notify_event_exit(process_t *process, + process_exit_code_t); + +#ifdef PROCESS_PRIVATE +MOCK_DECL(STATIC int, process_read_stdout, (process_t *, buf_t *)); +MOCK_DECL(STATIC int, process_read_stderr, (process_t *, buf_t *)); +MOCK_DECL(STATIC void, process_write_stdin, (process_t *, buf_t *)); + +STATIC void process_read_data(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_buffer(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +STATIC void process_read_lines(process_t *process, + buf_t *buffer, + process_read_callback_t callback); +#endif /* defined(PROCESS_PRIVATE). */ + +#endif /* defined(TOR_PROCESS_H). */ diff --git a/src/test/include.am b/src/test/include.am index e5eae56e25..482897c7a5 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -153,6 +153,7 @@ src_test_test_SOURCES += \ src/test/test_pem.c \ src/test/test_periodic_event.c \ src/test/test_policy.c \ + src/test/test_process.c \ src/test/test_procmon.c \ src/test/test_proto_http.c \ src/test/test_proto_misc.c \ diff --git a/src/test/test.c b/src/test/test.c index 17b736d305..b13c3ffbe2 100644 --- a/src/test/test.c +++ b/src/test/test.c @@ -898,6 +898,7 @@ struct testgroup_t testgroups[] = { { "periodic-event/" , periodic_event_tests }, { "policy/" , policy_tests }, { "procmon/", procmon_tests }, + { "process/", process_tests }, { "proto/http/", proto_http_tests }, { "proto/misc/", proto_misc_tests }, { "protover/", protover_tests }, diff --git a/src/test/test.h b/src/test/test.h index 092356f0fb..7c17389776 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -241,6 +241,7 @@ extern struct testcase_t pem_tests[]; extern struct testcase_t periodic_event_tests[]; extern struct testcase_t policy_tests[]; extern struct testcase_t procmon_tests[]; +extern struct testcase_t process_tests[]; extern struct testcase_t proto_http_tests[]; extern struct testcase_t proto_misc_tests[]; extern struct testcase_t protover_tests[]; diff --git a/src/test/test_process.c b/src/test/test_process.c new file mode 100644 index 0000000000..2adbde7ad2 --- /dev/null +++ b/src/test/test_process.c @@ -0,0 +1,558 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process.c + * \brief Test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "test/test.h" + +#define PROCESS_PRIVATE +#include "lib/process/process.h" + +static const char *stdout_read_buffer; +static const char *stderr_read_buffer; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static int +process_mocked_read_stdout(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stdout_read_buffer != NULL) { + buf_add_string(buffer, stdout_read_buffer); + stdout_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static int +process_mocked_read_stderr(process_t *process, buf_t *buffer) +{ + (void)process; + + if (stderr_read_buffer != NULL) { + buf_add_string(buffer, stderr_read_buffer); + stderr_read_buffer = NULL; + } + + return (int)buf_datalen(buffer); +} + +static void +process_mocked_write_stdin(process_t *process, buf_t *buffer) +{ + const size_t size = buf_datalen(buffer); + + if (size == 0) + return; + + char *data = tor_malloc_zero(size + 1); + process_data_t *process_data = process_get_data(process); + + buf_get_bytes(buffer, data, size); + smartlist_add(process_data->stdin_data, data); +} + +static void +process_stdout_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static void +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + done: + return; +} + +static void +test_default_values(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new("/path/to/nothing"); + + /* We are not running by default. */ + tt_int_op(PROCESS_STATUS_NOT_RUNNING, OP_EQ, process_get_status(process)); + + /* We use the line protocol by default. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + /* We don't set any custom data by default. */ + tt_ptr_op(NULL, OP_EQ, process_get_data(process)); + + /* Our command was given to the process_t's constructor in process_new(). */ + tt_str_op("/path/to/nothing", OP_EQ, process_get_command(process)); + + /* Make sure we are listed in the list of proccesses. */ + tt_assert(smartlist_contains(process_get_all_processes(), + process)); + + /* Our arguments should be empty. */ + tt_int_op(0, OP_EQ, + smartlist_len(process_get_arguments(process))); + + done: + process_free(process); + process_free_all(); +} + +static void +test_stringified_types(void *arg) +{ + (void)arg; + + /* process_protocol_t values. */ + tt_str_op("Raw", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_RAW)); + tt_str_op("Line", OP_EQ, process_protocol_to_string(PROCESS_PROTOCOL_LINE)); + + /* process_status_t values. */ + tt_str_op("not running", OP_EQ, + process_status_to_string(PROCESS_STATUS_NOT_RUNNING)); + tt_str_op("running", OP_EQ, + process_status_to_string(PROCESS_STATUS_RUNNING)); + tt_str_op("error", OP_EQ, + process_status_to_string(PROCESS_STATUS_ERROR)); + + done: + return; +} + +static void +test_line_protocol_simple(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\r\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_multi(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\r\nOnion Onion Onion\nA B C D\r\n\r\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\nFoo bar baz\nOnion Onion Onion\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(4, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Onion Onion Onion"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "A B C D"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + ""); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Foo bar baz"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Onion Onion Onion"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_line_protocol_partial(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the line protocol. */ + tt_int_op(PROCESS_PROTOCOL_LINE, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout this is a partial line ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr this is a partial line ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should NOT be ready. */ + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nAnother partial string goes here ..."; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = " the end\nFoo bar baz\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Some data should be ready. */ + tt_int_op(3, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(3, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "Foo bar baz"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr this is a partial line ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Another partial string goes here ... the end"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Foo bar baz"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_raw_protocol_simple(void *arg) +{ + (void)arg; + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_protocol(process, PROCESS_PROTOCOL_RAW); + + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + + MOCK(process_read_stdout, process_mocked_read_stdout); + MOCK(process_read_stderr, process_mocked_read_stderr); + + /* Make sure we are running with the raw protocol. */ + tt_int_op(PROCESS_PROTOCOL_RAW, OP_EQ, process_get_protocol(process)); + + tt_int_op(0, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(0, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello stdout\n"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello stderr\n"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stderr_data)); + + stdout_read_buffer = "Hello, again, stdout\nThis contains multiple lines"; + process_notify_event_stdout(process); + tt_ptr_op(NULL, OP_EQ, stdout_read_buffer); + + stderr_read_buffer = "Hello, again, stderr\nThis contains multiple lines"; + process_notify_event_stderr(process); + tt_ptr_op(NULL, OP_EQ, stderr_read_buffer); + + /* Data should be ready. */ + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdout_data)); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stderr_data)); + + /* Check if the data is correct. */ + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + "Hello stdout\n"); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "Hello, again, stdout\nThis contains multiple lines"); + + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Hello stderr\n"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "Hello, again, stderr\nThis contains multiple lines"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_read_stdout); + UNMOCK(process_read_stderr); +} + +static void +test_write_simple(void *arg) +{ + (void)arg; + + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + + MOCK(process_write_stdin, process_mocked_write_stdin); + + process_write(process, (uint8_t *)"Hello world\n", 12); + process_notify_event_stdin(process); + tt_int_op(1, OP_EQ, smartlist_len(process_data->stdin_data)); + + process_printf(process, "Hello %s !\n", "moon"); + process_notify_event_stdin(process); + tt_int_op(2, OP_EQ, smartlist_len(process_data->stdin_data)); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); + + UNMOCK(process_write_stdin); +} + +static void +test_exit_simple(void *arg) +{ + (void)arg; + + process_init(); + + process_data_t *process_data = process_data_new(); + + process_t *process = process_new(""); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Our default is 0. */ + tt_int_op(0, OP_EQ, process_data->exit_code); + + /* Fake that we are a running process. */ + process_set_status(process, PROCESS_STATUS_RUNNING); + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_RUNNING); + + /* Fake an exit. */ + process_notify_event_exit(process, 1337); + + /* Check if our state changed and if our callback fired. */ + tt_int_op(process_get_status(process), OP_EQ, PROCESS_STATUS_NOT_RUNNING); + tt_int_op(1337, OP_EQ, process_data->exit_code); + + done: + process_set_data(process, process_data); + process_data_free(process_data); + process_free(process); + process_free_all(); +} + +static void +test_argv_simple(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new("/bin/cat"); + char **argv = NULL; + + /* Setup some arguments. */ + process_append_argument(process, "foo"); + process_append_argument(process, "bar"); + process_append_argument(process, "baz"); + + /* Check the number of elements. */ + tt_int_op(3, OP_EQ, + smartlist_len(process_get_arguments(process))); + + /* Let's try to convert it into a Unix style char **argv. */ + argv = process_get_argv(process); + + /* Check our values. */ + tt_str_op(argv[0], OP_EQ, "/bin/cat"); + tt_str_op(argv[1], OP_EQ, "foo"); + tt_str_op(argv[2], OP_EQ, "bar"); + tt_str_op(argv[3], OP_EQ, "baz"); + tt_ptr_op(argv[4], OP_EQ, NULL); + + done: + tor_free(argv); + process_free(process); + process_free_all(); +} + +struct testcase_t process_tests[] = { + { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, + { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, + { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, + { "line_protocol_partial", test_line_protocol_partial, TT_FORK, NULL, NULL }, + { "raw_protocol_simple", test_raw_protocol_simple, TT_FORK, NULL, NULL }, + { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, + { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, + { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; From 2e957027e28449d4c3254cc404d154f4bce41bfc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:43:27 +0100 Subject: [PATCH 09/34] Add Unix backend for the Process subsystem. This patch adds the Unix backend for the Process subsystem. The Unix backend attaches file descriptors from the child process's standard in, out and error to Tor's libevent based main loop using traditional Unix pipes. We use the already available `waitpid` module to get events whenever the child process terminates. See: https://bugs.torproject.org/28179 --- src/lib/process/include.am | 2 + src/lib/process/process.c | 43 +++ src/lib/process/process.h | 5 + src/lib/process/process_unix.c | 611 +++++++++++++++++++++++++++++++++ src/lib/process/process_unix.h | 64 ++++ src/test/test_process.c | 21 ++ 6 files changed, 746 insertions(+) create mode 100644 src/lib/process/process_unix.c create mode 100644 src/lib/process/process_unix.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index 1d213d1c00..f068032681 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -10,6 +10,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/env.c \ src/lib/process/pidfile.c \ src/lib/process/process.c \ + src/lib/process/process_unix.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -26,6 +27,7 @@ noinst_HEADERS += \ src/lib/process/env.h \ src/lib/process/pidfile.h \ src/lib/process/process.h \ + src/lib/process/process_unix.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index d3967a52da..8d6a9d3fa0 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -15,6 +15,7 @@ #include "lib/log/log.h" #include "lib/log/util_bug.h" #include "lib/process/process.h" +#include "lib/process/process_unix.h" #include "lib/process/env.h" #ifdef HAVE_STDDEF_H @@ -64,6 +65,11 @@ struct process_t { /** Do we need to store some custom data with the process? */ void *data; + +#ifndef _WIN32 + /** Our Unix process handle. */ + process_unix_t *unix_process; +#endif }; /** Convert a given process status in status to its string @@ -161,6 +167,11 @@ process_new(const char *command) process->stderr_buffer = buf_new(); process->stdin_buffer = buf_new(); +#ifndef _WIN32 + /* Prepare our Unix process handle. */ + process->unix_process = process_unix_new(); +#endif + smartlist_add(processes, process); return process; @@ -188,6 +199,11 @@ process_free_(process_t *process) buf_free(process->stderr_buffer); buf_free(process->stdin_buffer); +#ifndef _WIN32 + /* Cleanup our Unix process handle. */ + process_unix_free(process->unix_process); +#endif + smartlist_remove(processes, process); tor_free(process); @@ -204,6 +220,10 @@ process_exec(process_t *process) log_info(LD_PROCESS, "Starting new process: %s", process->command); +#ifndef _WIN32 + status = process_unix_exec(process); +#endif + /* Update our state. */ process_set_status(process, status); @@ -391,6 +411,17 @@ process_get_environment(const process_t *process) return process_environment_make(process->environment); } +#ifndef _WIN32 +/** Get the internal handle for the Unix backend. */ +process_unix_t * +process_get_unix_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->unix_process); + return process->unix_process; +} +#endif + /** Write size bytes of data to the given process's standard * input. */ void @@ -510,7 +541,11 @@ MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) tor_assert(process); tor_assert(buffer); +#ifndef _WIN32 + return process_unix_read_stdout(process, buffer); +#else return 0; +#endif } /** This function is called whenever the Process backend have notified us that @@ -521,7 +556,11 @@ MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) tor_assert(process); tor_assert(buffer); +#ifndef _WIN32 + return process_unix_read_stderr(process, buffer); +#else return 0; +#endif } /** This function calls the backend function for the given process whenever @@ -531,6 +570,10 @@ MOCK_IMPL(STATIC void, process_write_stdin, { tor_assert(process); tor_assert(buffer); + +#ifndef _WIN32 + process_unix_write(process, buffer); +#endif } /** This function calls the protocol handlers based on the value of diff --git a/src/lib/process/process.h b/src/lib/process/process.h index cf20a5d80b..b17b8dac7c 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -95,6 +95,11 @@ void *process_get_data(const process_t *process); void process_set_status(process_t *process, process_status_t status); process_status_t process_get_status(const process_t *process); +#ifndef _WIN32 +struct process_unix_t; +struct process_unix_t *process_get_unix_process(const process_t *process); +#endif + void process_write(process_t *process, const uint8_t *data, size_t size); void process_vprintf(process_t *process, diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c new file mode 100644 index 0000000000..c3691f1851 --- /dev/null +++ b/src/lib/process/process_unix.c @@ -0,0 +1,611 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.c + * \brief Module for working with Unix processes. + **/ + +#define PROCESS_UNIX_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/process/process.h" +#include "lib/process/process_unix.h" +#include "lib/process/waitpid.h" +#include "lib/process/env.h" + +#include + +#ifdef HAVE_STRING_H +#include +#endif + +#ifdef HAVE_ERRNO_H +#include +#endif + +#ifdef HAVE_UNISTD_H +#include +#endif + +#ifdef HAVE_FCNTL_H +#include +#endif + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) +#include +#endif + +#if HAVE_SIGNAL_H +#include +#endif + +#ifndef _WIN32 + +/** Maximum number of file descriptors, if we cannot get it via sysconf() */ +#define DEFAULT_MAX_FD 256 + +/** Internal state for Unix handles. */ +struct process_unix_handle_t { + /** Unix File Descriptor. */ + int fd; + + /** Have we reached end of file? */ + bool reached_eof; + + /** Event structure for libevent. */ + struct event *event; + + /** Are we writing? */ + bool is_writing; +}; + +/** Internal state for our Unix process. */ +struct process_unix_t { + /** Standard in handle. */ + process_unix_handle_t stdin_handle; + + /** Standard out handle. */ + process_unix_handle_t stdout_handle; + + /** Standard error handle. */ + process_unix_handle_t stderr_handle; + + /** The process identifier of our process. */ + pid_t pid; + + /** Waitpid Callback structure. */ + waitpid_callback_t *waitpid; +}; + +/** Returns a newly allocated process_unix_t. */ +process_unix_t * +process_unix_new(void) +{ + process_unix_t *unix_process; + unix_process = tor_malloc_zero(sizeof(process_unix_t)); + + unix_process->stdin_handle.fd = -1; + unix_process->stderr_handle.fd = -1; + unix_process->stdout_handle.fd = -1; + + return unix_process; +} + +/** Deallocates the given unix_process. */ +void +process_unix_free_(process_unix_t *unix_process) +{ + if (! unix_process) + return; + + /* Clean up our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + + /* FIXME(ahf): Refactor waitpid code? */ + unix_process->waitpid = NULL; + + /* Cleanup our events. */ + if (! unix_process->stdout_handle.reached_eof) + process_unix_stop_reading(&unix_process->stdout_handle); + + if (! unix_process->stderr_handle.reached_eof) + process_unix_stop_reading(&unix_process->stderr_handle); + + process_unix_stop_writing(&unix_process->stdin_handle); + + tor_event_free(unix_process->stdout_handle.event); + tor_event_free(unix_process->stderr_handle.event); + tor_event_free(unix_process->stdin_handle.event); + + tor_free(unix_process); +} + +/** Executes the given process as a child process of Tor. This function is + * responsible for setting up the child process and run it. This includes + * setting up pipes for interprocess communication, initialize the waitpid + * callbacks, and finally run fork() followed by execve(). Returns + * PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_unix_exec(process_t *process) +{ + static int max_fd = -1; + + process_unix_t *unix_process; + pid_t pid; + int stdin_pipe[2]; + int stdout_pipe[2]; + int stderr_pipe[2]; + int retval, fd; + + unix_process = process_get_unix_process(process); + + /* Create standard in pipe. */ + retval = pipe(stdin_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdin " + "communication with process: %s", + strerror(errno)); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard out pipe. */ + retval = pipe(stdout_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stdout " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Create standard error pipe. */ + retval = pipe(stderr_pipe); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Unable to create pipe for stderr " + "communication with process: %s", + strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + +#ifdef _SC_OPEN_MAX + if (-1 == max_fd) { + max_fd = (int)sysconf(_SC_OPEN_MAX); + + if (max_fd == -1) { + max_fd = DEFAULT_MAX_FD; + log_warn(LD_PROCESS, + "Cannot find maximum file descriptor, assuming: %d", max_fd); + } + } +#else /* !(defined(_SC_OPEN_MAX)) */ + max_fd = DEFAULT_MAX_FD; +#endif /* defined(_SC_OPEN_MAX) */ + + pid = fork(); + + if (0 == pid) { + /* This code is running in the child process context. */ + +#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) + /* Attempt to have the kernel issue a SIGTERM if the parent + * goes away. Certain attributes of the binary being execve()ed + * will clear this during the execve() call, but it's better + * than nothing. + */ + prctl(PR_SET_PDEATHSIG, SIGTERM); +#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ + + /* Link process stdout to the write end of the pipe. */ + retval = dup2(stdout_pipe[1], STDOUT_FILENO); + if (-1 == retval) + goto error; + + /* Link process stderr to the write end of the pipe. */ + retval = dup2(stderr_pipe[1], STDERR_FILENO); + if (-1 == retval) + goto error; + + /* Link process stdin to the read end of the pipe */ + retval = dup2(stdin_pipe[0], STDIN_FILENO); + if (-1 == retval) + goto error; + + /* Close our pipes now after they have been dup2()'ed. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + close(stdout_pipe[0]); + close(stdout_pipe[1]); + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /* Close all other fds, including the read end of the pipe. XXX: We should + * now be doing enough FD_CLOEXEC setting to make this needless. + */ + for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) + close(fd); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Call the requested program. */ + retval = execve(argv[0], argv, env->unixoid_environment_block); + + /* If we made it here it is because execve failed :-( */ + if (-1 == retval) + fprintf(stderr, "Call to execve() failed: %s", strerror(errno)); + + tor_free(argv); + process_environment_free(env); + + tor_assert_unreached(); + + error: + /* LCOV_EXCL_START */ + fprintf(stderr, "Error from child process: %s", strerror(errno)); + _exit(1); + /* LCOV_EXCL_STOP */ + } + + /* We are in the parent process. */ + if (-1 == pid) { + log_warn(LD_PROCESS, + "Failed to create child process: %s", strerror(errno)); + + /** Cleanup standard in pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard out pipe. */ + close(stdin_pipe[0]); + close(stdin_pipe[1]); + + /** Cleanup standard error pipe. */ + close(stderr_pipe[0]); + close(stderr_pipe[1]); + + return PROCESS_STATUS_ERROR; + } + + /* Register our PID. */ + unix_process->pid = pid; + + /* Setup waitpid callbacks. */ + unix_process->waitpid = set_waitpid_callback(pid, + process_unix_waitpid_callback, + process); + + /* Handle standard out. */ + unix_process->stdout_handle.fd = stdout_pipe[0]; + retval = close(stdout_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close write end of standard out pipe: %s", + strerror(errno)); + } + + /* Handle standard error. */ + unix_process->stderr_handle.fd = stderr_pipe[0]; + retval = close(stderr_pipe[1]); + + if (-1 == retval) { + log_warn(LD_PROCESS, + "Failed to close write end of standard error pipe: %s", + strerror(errno)); + } + + /* Handle standard in. */ + unix_process->stdin_handle.fd = stdin_pipe[1]; + retval = close(stdin_pipe[0]); + + if (-1 == retval) { + log_warn(LD_PROCESS, "Failed to close read end of standard in pipe: %s", + strerror(errno)); + } + + /* Setup our handles. */ + process_unix_setup_handle(process, + &unix_process->stdout_handle, + EV_READ|EV_PERSIST, + stdout_read_callback); + + process_unix_setup_handle(process, + &unix_process->stderr_handle, + EV_READ|EV_PERSIST, + stderr_read_callback); + + process_unix_setup_handle(process, + &unix_process->stdin_handle, + EV_WRITE|EV_PERSIST, + stdin_write_callback); + + /* Start reading from standard out and standard error. */ + process_unix_start_reading(&unix_process->stdout_handle); + process_unix_start_reading(&unix_process->stderr_handle); + + return PROCESS_STATUS_RUNNING; +} + +/** Write the given buffer as input to the given process's + * standard input. Returns the number of bytes written. */ +int +process_unix_write(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + size_t buffer_flush_len = buf_datalen(buffer); + const size_t max_to_write = MIN(PROCESS_MAX_WRITE, buffer_flush_len); + + /* If we have data to write (when buffer_flush_len > 0) and we are not + * currently getting file descriptor events from the kernel, we tell the + * kernel to start notifying us about when we can write to our file + * descriptor and return. */ + if (buffer_flush_len > 0 && ! unix_process->stdin_handle.is_writing) { + process_unix_start_writing(&unix_process->stdin_handle); + return 0; + } + + /* We don't have any data to write, but the kernel is currently notifying us + * about whether we are able to write or not. Tell the kernel to stop + * notifying us until we have data to write. */ + if (buffer_flush_len == 0 && unix_process->stdin_handle.is_writing) { + process_unix_stop_writing(&unix_process->stdin_handle); + return 0; + } + + /* We have data to write and the kernel have told us to write it. */ + return buf_flush_to_pipe(buffer, + process_get_unix_process(process)->stdin_handle.fd, + max_to_write, &buffer_flush_len); +} + +/** Read data from the given process's standard output and put it into + * buffer. Returns the number of bytes read. */ +int +process_unix_read_stdout(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stdout_handle, + buffer); +} + +/** Read data from the given process's standard error and put it into + * buffer. Returns the number of bytes read. */ +int +process_unix_read_stderr(process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_unix_t *unix_process = process_get_unix_process(process); + + return process_unix_read_handle(process, + &unix_process->stderr_handle, + buffer); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard output. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stdout_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdout(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * read from the child process's standard error. We notify the Process + * subsystem, which is then responsible for calling back to us for doing the + * actual reading of the data. */ +STATIC void +stderr_read_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stderr(process); +} + +/** This function is called whenever libevent thinks we have data that could be + * written the child process's standard input. We notify the Process subsystem, + * which is then responsible for calling back to us for doing the actual write + * of the data. */ +STATIC void +stdin_write_callback(evutil_socket_t fd, short event, void *data) +{ + (void)fd; + (void)event; + + process_t *process = data; + tor_assert(process); + + process_notify_event_stdin(process); +} + +/** This function tells libevent that we are interested in receiving read + * events from the given handle. */ +STATIC void +process_unix_start_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); +} + +/** This function tells libevent that we are no longer interested in receiving + * read events from the given handle. */ +STATIC void +process_unix_stop_reading(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); +} + +/** This function tells libevent that we are interested in receiving write + * events from the given handle. */ +STATIC void +process_unix_start_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (event_add(handle->event, NULL)) + log_warn(LD_PROCESS, + "Unable to add libevent event for handle."); + + handle->is_writing = true; +} + +/** This function tells libevent that we are no longer interested in receiving + * write events from the given handle. */ +STATIC void +process_unix_stop_writing(process_unix_handle_t *handle) +{ + tor_assert(handle); + + if (handle->event == NULL) + return; + + if (event_del(handle->event)) + log_warn(LD_PROCESS, + "Unable to delete libevent event for handle."); + + handle->is_writing = false; +} + +/** This function is called when the waitpid system have detected that our + * process have terminated. We disable the waitpid system and notify the + * Process subsystem that we have terminated. */ +STATIC void +process_unix_waitpid_callback(int status, void *data) +{ + tor_assert(data); + + process_t *process = data; + process_unix_t *unix_process = process_get_unix_process(process); + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Remove our waitpid callback. */ + clear_waitpid_callback(unix_process->waitpid); + unix_process->waitpid = NULL; +} + +/** This function sets the file descriptor in the handle as non-blocking + * and configures the libevent event structure based on the given flags + * to ensure that callback is called whenever we have events on the + * given handle. */ +STATIC void +process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(callback); + + /* Put our file descriptor into non-blocking mode. */ + if (fcntl(handle->fd, F_SETFL, O_NONBLOCK) < 0) { + log_warn(LD_PROCESS, "Unable mark Unix handle as non-blocking: %s", + strerror(errno)); + } + + /* Setup libevent event. */ + handle->event = tor_event_new(tor_libevent_get_base(), + handle->fd, + flags, + callback, + process); +} + +/** This function reads data from the given handle and puts it into + * buffer. Returns the number of bytes read this way. */ +STATIC int +process_unix_read_handle(process_t *process, + process_unix_handle_t *handle, + buf_t *buffer) +{ + tor_assert(process); + tor_assert(handle); + tor_assert(buffer); + + int ret = 0; + int eof = 0; + int error = 0; + + ret = buf_read_from_pipe(buffer, + handle->fd, + PROCESS_MAX_READ, + &eof, + &error); + + if (error) + log_warn(LD_PROCESS, + "Unable to read data: %s", strerror(error)); + + if (eof) { + handle->reached_eof = true; + process_unix_stop_reading(handle); + } + + return ret; +} + +#endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h new file mode 100644 index 0000000000..5fc23bcf07 --- /dev/null +++ b/src/lib/process/process_unix.h @@ -0,0 +1,64 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_unix.h + * \brief Header for process_unix.c + **/ + +#ifndef TOR_PROCESS_UNIX_H +#define TOR_PROCESS_UNIX_H + +#ifndef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" + +#include + +struct process_t; + +struct process_unix_t; +typedef struct process_unix_t process_unix_t; + +process_unix_t *process_unix_new(void); +void process_unix_free_(process_unix_t *unix_process); +#define process_unix_free(s) \ + FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) + +process_status_t process_unix_exec(struct process_t *process); + +int process_unix_write(struct process_t *process, buf_t *buffer); +int process_unix_read_stdout(struct process_t *process, buf_t *buffer); +int process_unix_read_stderr(struct process_t *process, buf_t *buffer); + +#ifdef PROCESS_UNIX_PRIVATE +struct process_unix_handle_t; +typedef struct process_unix_handle_t process_unix_handle_t; + +STATIC void stdout_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stderr_read_callback(evutil_socket_t fd, short event, void *data); +STATIC void stdin_write_callback(evutil_socket_t fd, short event, void *data); + +STATIC void process_unix_start_reading(process_unix_handle_t *); +STATIC void process_unix_stop_reading(process_unix_handle_t *); + +STATIC void process_unix_start_writing(process_unix_handle_t *); +STATIC void process_unix_stop_writing(process_unix_handle_t *); + +STATIC void process_unix_waitpid_callback(int status, void *data); + +STATIC void process_unix_setup_handle(process_t *process, + process_unix_handle_t *handle, + short flags, + event_callback_fn callback); +STATIC int process_unix_read_handle(process_t *, + process_unix_handle_t *, + buf_t *); +#endif /* defined(PROCESS_UNIX_PRIVATE). */ + +#endif /* defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_UNIX_H). */ diff --git a/src/test/test_process.c b/src/test/test_process.c index 2adbde7ad2..816695ccab 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -12,6 +12,8 @@ #define PROCESS_PRIVATE #include "lib/process/process.h" +#define PROCESS_UNIX_PRIVATE +#include "lib/process/process_unix.h" static const char *stdout_read_buffer; static const char *stderr_read_buffer; @@ -544,6 +546,24 @@ test_argv_simple(void *arg) process_free_all(); } +static void +test_unix(void *arg) +{ + (void)arg; +#ifndef _WIN32 + process_init(); + + process_t *process = process_new(); + + /* On Unix all processes should have a Unix process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_unix_process(process)); + + done: + process_free(process); + process_free_all(); +#endif +} + struct testcase_t process_tests[] = { { "default_values", test_default_values, TT_FORK, NULL, NULL }, { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, @@ -554,5 +574,6 @@ struct testcase_t process_tests[] = { { "write_simple", test_write_simple, TT_FORK, NULL, NULL }, { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, + { "unix", test_unix, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; From bb784cf4f36256a8276ba60641d3ff766b9cd9df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 04:49:23 +0100 Subject: [PATCH 10/34] Add Windows backend for the Process subsystem. This patch adds support for Microsoft Windows in the Process subsystem. Libevent does not support mixing different types of handles (sockets, named pipes, etc.) on Windows in its core event loop code. This have historically meant that Tor have avoided attaching any non-networking handles to the event loop. This patch uses a slightly different approach to roughly support the same features for the Process subsystem as we do with the Unix backend. In this patch we use Windows Extended I/O functions (ReadFileEx() and WriteFileEx()) which executes asynchronously in the background and executes a completion routine when the scheduled read or write operation have completed. This is much different from the Unix backend where the operating system signals to us whenever a file descriptor is "ready" to either being read from or written to. To make the Windows operating system execute the completion routines of ReadFileEx() and WriteFileEx() we must get the Tor process into what Microsoft calls an "alertable" state. To do this we execute SleepEx() with a zero millisecond sleep time from a main loop timer that ticks once a second. This moves the process into the "alertable" state and when we return from the zero millisecond timeout all the outstanding I/O completion routines will be called and we can schedule the next reads and writes. The timer loop is also responsible for detecting whether our child processes have terminated since the last timer tick. See: https://bugs.torproject.org/28179 --- src/lib/process/include.am | 2 + src/lib/process/process.c | 35 +- src/lib/process/process.h | 3 + src/lib/process/process_win32.c | 857 ++++++++++++++++++++++++++++++++ src/lib/process/process_win32.h | 90 ++++ src/test/test_process.c | 23 +- 6 files changed, 1007 insertions(+), 3 deletions(-) create mode 100644 src/lib/process/process_win32.c create mode 100644 src/lib/process/process_win32.h diff --git a/src/lib/process/include.am b/src/lib/process/include.am index f068032681..aa73356146 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -11,6 +11,7 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/pidfile.c \ src/lib/process/process.c \ src/lib/process/process_unix.c \ + src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ src/lib/process/subprocess.c \ @@ -28,6 +29,7 @@ noinst_HEADERS += \ src/lib/process/pidfile.h \ src/lib/process/process.h \ src/lib/process/process_unix.h \ + src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ src/lib/process/subprocess.h \ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 8d6a9d3fa0..d4237b2b1f 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -16,6 +16,7 @@ #include "lib/log/util_bug.h" #include "lib/process/process.h" #include "lib/process/process_unix.h" +#include "lib/process/process_win32.h" #include "lib/process/env.h" #ifdef HAVE_STDDEF_H @@ -69,6 +70,9 @@ struct process_t { #ifndef _WIN32 /** Our Unix process handle. */ process_unix_t *unix_process; +#else + /** Our Win32 process handle. */ + process_win32_t *win32_process; #endif }; @@ -117,6 +121,10 @@ void process_init(void) { processes = smartlist_new(); + +#ifdef _WIN32 + process_win32_init(); +#endif } /** Free up all resources that is handled by the Process subsystem. Note that @@ -124,6 +132,10 @@ process_init(void) void process_free_all(void) { +#ifdef _WIN32 + process_win32_deinit(); +#endif + SMARTLIST_FOREACH(processes, process_t *, x, process_free(x)); smartlist_free(processes); } @@ -170,6 +182,9 @@ process_new(const char *command) #ifndef _WIN32 /* Prepare our Unix process handle. */ process->unix_process = process_unix_new(); +#else + /* Prepare our Win32 process handle. */ + process->win32_process = process_win32_new(); #endif smartlist_add(processes, process); @@ -202,6 +217,9 @@ process_free_(process_t *process) #ifndef _WIN32 /* Cleanup our Unix process handle. */ process_unix_free(process->unix_process); +#else + /* Cleanup our Win32 process handle. */ + process_win32_free(process->win32_process); #endif smartlist_remove(processes, process); @@ -222,6 +240,8 @@ process_exec(process_t *process) #ifndef _WIN32 status = process_unix_exec(process); +#else + status = process_win32_exec(process); #endif /* Update our state. */ @@ -420,6 +440,15 @@ process_get_unix_process(const process_t *process) tor_assert(process->unix_process); return process->unix_process; } +#else +/** Get the internal handle for Windows backend. */ +process_win32_t * +process_get_win32_process(const process_t *process) +{ + tor_assert(process); + tor_assert(process->win32_process); + return process->win32_process; +} #endif /** Write size bytes of data to the given process's standard @@ -544,7 +573,7 @@ MOCK_IMPL(STATIC int, process_read_stdout, (process_t *process, buf_t *buffer)) #ifndef _WIN32 return process_unix_read_stdout(process, buffer); #else - return 0; + return process_win32_read_stdout(process, buffer); #endif } @@ -559,7 +588,7 @@ MOCK_IMPL(STATIC int, process_read_stderr, (process_t *process, buf_t *buffer)) #ifndef _WIN32 return process_unix_read_stderr(process, buffer); #else - return 0; + return process_win32_read_stderr(process, buffer); #endif } @@ -573,6 +602,8 @@ MOCK_IMPL(STATIC void, process_write_stdin, #ifndef _WIN32 process_unix_write(process, buffer); +#else + process_win32_write(process, buffer); #endif } diff --git a/src/lib/process/process.h b/src/lib/process/process.h index b17b8dac7c..f759c71939 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -98,6 +98,9 @@ process_status_t process_get_status(const process_t *process); #ifndef _WIN32 struct process_unix_t; struct process_unix_t *process_get_unix_process(const process_t *process); +#else +struct process_win32_t; +struct process_win32_t *process_get_win32_process(const process_t *process); #endif void process_write(process_t *process, diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c new file mode 100644 index 0000000000..a019e0b4f3 --- /dev/null +++ b/src/lib/process/process_win32.c @@ -0,0 +1,857 @@ +/* Copyright (c) 2003, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.c + * \brief Module for working with Windows processes. + **/ + +#define PROCESS_WIN32_PRIVATE +#include "lib/intmath/cmp.h" +#include "lib/container/buffers.h" +#include "lib/net/buffers_net.h" +#include "lib/container/smartlist.h" +#include "lib/log/log.h" +#include "lib/log/util_bug.h" +#include "lib/log/win32err.h" +#include "lib/process/process.h" +#include "lib/process/process_win32.h" +#include "lib/process/subprocess.h" +#include "lib/process/env.h" + +#ifdef HAVE_SYS_TIME_H +#include +#endif + +#ifdef _WIN32 + +/** The size of our intermediate buffers. */ +#define BUFFER_SIZE (1024) + +/** Timer that ticks once a second and calls the process_win32_timer_callback() + * function. */ +static periodic_timer_t *periodic_timer; + +/** Structure to represent the state around the pipe HANDLE. + * + * This structure is used to store state about a given HANDLE, including + * whether we have reached end of file, its intermediate buffers, and how much + * data that is available in the intermediate buffer. */ +struct process_win32_handle_t { + /** Standard out pipe handle. */ + HANDLE pipe; + + /** True iff we have reached EOF from the pipe. */ + bool reached_eof; + + /** How much data is available in buffer. */ + size_t data_available; + + /** Intermediate buffer for ReadFileEx() and WriteFileEx(). */ + char buffer[BUFFER_SIZE]; + + /** Overlapped structure for ReadFileEx() and WriteFileEx(). */ + OVERLAPPED overlapped; + + /** Are we waiting for another I/O operation to complete? */ + bool busy; +}; + +/** Structure to represent the Windows specific implementation details of this + * Process backend. + * + * This structure is attached to process_t (see process.h) and is + * reachable from process_t via the process_get_win32_process() + * method. */ +struct process_win32_t { + /** Standard in state. */ + process_win32_handle_t stdin_handle; + + /** Standard out state. */ + process_win32_handle_t stdout_handle; + + /** Standard error state. */ + process_win32_handle_t stderr_handle; + + /** Process Information. */ + PROCESS_INFORMATION process_information; +}; + +/** Create a new process_win32_t. + * + * This function constructs a new process_win32_t and initializes the + * default values. */ +process_win32_t * +process_win32_new(void) +{ + process_win32_t *win32_process; + win32_process = tor_malloc_zero(sizeof(process_win32_t)); + + win32_process->stdin_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stdout_handle.pipe = INVALID_HANDLE_VALUE; + win32_process->stderr_handle.pipe = INVALID_HANDLE_VALUE; + + return win32_process; +} + +/** Free a given process_win32_t. + * + * This function deinitializes and frees up the resources allocated for the + * given process_win32_t. */ +void +process_win32_free_(process_win32_t *win32_process) +{ + if (! win32_process) + return; + + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + + tor_free(win32_process); +} + +/** Initialize the Windows backend of the Process subsystem. */ +void +process_win32_init(void) +{ + /* We don't start the periodic timer here because it makes no sense to have + * the timer running until we have some processes that benefits from the + * timer timer ticks. */ +} + +/** Deinitialize the Windows backend of the Process subsystem. */ +void +process_win32_deinit(void) +{ + /* Stop our timer, but only if it's running. */ + if (process_win32_timer_running()) + process_win32_timer_stop(); +} + +/** Execute the given process. This function is responsible for setting up + * named pipes for I/O between the child process and the Tor process. Returns + * PROCESS_STATUS_RUNNING upon success. */ +process_status_t +process_win32_exec(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + HANDLE stdout_pipe_read = NULL; + HANDLE stdout_pipe_write = NULL; + HANDLE stderr_pipe_read = NULL; + HANDLE stderr_pipe_write = NULL; + HANDLE stdin_pipe_read = NULL; + HANDLE stdin_pipe_write = NULL; + BOOL ret = FALSE; + const char *filename = process_get_command(process); + + /* Not much we can do if we haven't been told what to start. */ + if (BUG(filename == NULL)) + return PROCESS_STATUS_ERROR; + + /* Setup our security attributes. */ + SECURITY_ATTRIBUTES security_attributes; + security_attributes.nLength = sizeof(security_attributes); + security_attributes.bInheritHandle = TRUE; + /* FIXME: should we set explicit security attributes? + * (See Ticket #2046, comment 5) */ + security_attributes.lpSecurityDescriptor = NULL; + + /* Create our standard out pipe. */ + if (! process_win32_create_pipe(&stdout_pipe_read, + &stdout_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create our standard error pipe. */ + if (! process_win32_create_pipe(&stderr_pipe_read, + &stderr_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_READER)) { + return PROCESS_STATUS_ERROR; + } + + /* Create out standard in pipe. */ + if (! process_win32_create_pipe(&stdin_pipe_read, + &stdin_pipe_write, + &security_attributes, + PROCESS_WIN32_PIPE_TYPE_WRITER)) { + return PROCESS_STATUS_ERROR; + } + + /* Configure startup info for our child process. */ + STARTUPINFOA startup_info; + + memset(&startup_info, 0, sizeof(startup_info)); + startup_info.cb = sizeof(startup_info); + startup_info.hStdError = stderr_pipe_write; + startup_info.hStdOutput = stdout_pipe_write; + startup_info.hStdInput = stdin_pipe_read; + startup_info.dwFlags |= STARTF_USESTDHANDLES; + + /* Create the env value for our new process. */ + process_environment_t *env = process_get_environment(process); + + /* Create the argv value for our new process. */ + char **argv = process_get_argv(process); + + /* Windows expects argv to be a whitespace delimited string, so join argv up + */ + char *joined_argv = tor_join_win_cmdline((const char **)argv); + + /* Create the child process */ + ret = CreateProcessA(filename, + joined_argv, + NULL, + NULL, + TRUE, + CREATE_NO_WINDOW, + env->windows_environment_block[0] == '\0' ? + NULL : env->windows_environment_block, + NULL, + &startup_info, + &win32_process->process_information); + + tor_free(argv); + tor_free(joined_argv); + process_environment_free(env); + + if (! ret) { + log_warn(LD_PROCESS, "CreateProcessA() failed: %s", + format_win32_error(GetLastError())); + + /* Cleanup our handles. */ + CloseHandle(stdout_pipe_read); + CloseHandle(stdout_pipe_write); + CloseHandle(stderr_pipe_read); + CloseHandle(stderr_pipe_write); + CloseHandle(stdin_pipe_read); + CloseHandle(stdin_pipe_write); + + return PROCESS_STATUS_ERROR; + } + + /* TODO: Should we close hProcess and hThread in + * process_handle->process_information? */ + win32_process->stdout_handle.pipe = stdout_pipe_read; + win32_process->stderr_handle.pipe = stderr_pipe_read; + win32_process->stdin_handle.pipe = stdin_pipe_write; + + /* Used by the callback functions from ReadFileEx() and WriteFileEx() such + * that we can figure out which process_t that was responsible for the event. + * + * Warning, here be dragons: + * + * MSDN says that the hEvent member of the overlapped structure is unused + * for ReadFileEx() and WriteFileEx, which allows us to store a pointer to + * our process state there. + */ + win32_process->stdout_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stderr_handle.overlapped.hEvent = (HANDLE)process; + win32_process->stdin_handle.overlapped.hEvent = (HANDLE)process; + + /* Start our timer if it is not already running. */ + if (! process_win32_timer_running()) + process_win32_timer_start(); + + /* We use Windows Extended I/O functions, so our completion callbacks are + * called automatically for us when there is data to read. Because of this + * we start the read of standard out and error right away. */ + process_notify_event_stdout(process); + process_notify_event_stderr(process); + + return PROCESS_STATUS_RUNNING; +} + +/** Schedule an async write of the data found in buffer for the given + * process. This function runs an async write operation of the content of + * buffer, if we are not already waiting for a pending I/O request. Returns the + * number of bytes that Windows will hopefully write for us in the background. + * */ +int +process_win32_write(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + const size_t buffer_size = buf_datalen(buffer); + + /* Windows is still writing our buffer. */ + if (win32_process->stdin_handle.busy) + return 0; + + /* Nothing for us to do right now. */ + if (buffer_size == 0) + return 0; + + /* We have reached end of file already? */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return 0; + + /* Figure out how much data we should read. */ + const size_t write_size = MIN(buffer_size, + sizeof(win32_process->stdin_handle.buffer)); + + /* Read data from the process_t buffer into our intermediate buffer. */ + buf_get_bytes(buffer, win32_process->stdin_handle.buffer, write_size); + + /* Schedule our write. */ + ret = WriteFileEx(win32_process->stdin_handle.pipe, + win32_process->stdin_handle.buffer, + write_size, + &win32_process->stdin_handle.overlapped, + process_win32_stdin_write_done); + + if (! ret) { + log_warn(LD_PROCESS, "WriteFileEx() failed: %s", + format_win32_error(GetLastError())); + return 0; + } + + /* This cast should be safe since our buffer can maximum be BUFFER_SIZE + * large. */ + return (int)write_size; +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard output of the child process + * and asks Windows to call process_win32_stdout_read_done() when that + * operation is finished. Returns the number of bytes moved into buffer. + * */ +int +process_win32_read_stdout(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stdout_handle, + buffer, + process_win32_stdout_read_done); +} + +/** This function is called from the Process subsystem whenever the Windows + * backend says it has data ready. This function also ensures that we are + * starting a new background read from the standard error of the child process + * and asks Windows to call process_win32_stderr_read_done() when that + * operation is finished. Returns the number of bytes moved into buffer. + * */ +int +process_win32_read_stderr(struct process_t *process, buf_t *buffer) +{ + tor_assert(process); + tor_assert(buffer); + + process_win32_t *win32_process = process_get_win32_process(process); + + return process_win32_read_from_handle(&win32_process->stderr_handle, + buffer, + process_win32_stderr_read_done); +} + +/** This function is responsible for moving the Tor process into what Microsoft + * calls an "alertable" state. Once the process is in an alertable state the + * Windows kernel will notify us when our background I/O requests have finished + * and the callbacks will be executed. */ +void +process_win32_trigger_completion_callbacks(void) +{ + DWORD ret; + + /* The call to SleepEx(dwMilliseconds, dwAlertable) makes the process sleep + * for dwMilliseconds and if dwAlertable is set to TRUE it will also cause + * the process to enter alertable state, where the Windows kernel will notify + * us about completed I/O requests from ReadFileEx() and WriteFileEX(), which + * will cause our completion callbacks to be executed. + * + * This function returns 0 if the time interval expired or WAIT_IO_COMPLETION + * if one or more I/O callbacks were executed. */ + ret = SleepEx(0, TRUE); + + /* Warn us if the function returned something we did not anticipate. */ + if (ret != 0 && ret != WAIT_IO_COMPLETION) { + log_warn(LD_PROCESS, "SleepEx() returned %lu", ret); + } +} + +/** Start the periodic timer which is reponsible for checking whether processes + * are still alive and to make sure that the Tor process is periodically being + * moved into an alertable state. */ +STATIC void +process_win32_timer_start(void) +{ + /* Make sure we never start our timer if it's already running. */ + if (BUG(process_win32_timer_running())) + return; + + /* Wake up once a second. */ + static const struct timeval interval = {1, 0}; + + log_info(LD_PROCESS, "Starting Windows Process I/O timer"); + periodic_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + process_win32_timer_callback, + NULL); +} + +/** Stops the periodic timer. */ +STATIC void +process_win32_timer_stop(void) +{ + if (BUG(periodic_timer == NULL)) + return; + + log_info(LD_PROCESS, "Stopping Windows Process I/O timer"); + periodic_timer_free(periodic_timer); +} + +/** Returns true iff the periodic timer is running. */ +STATIC bool +process_win32_timer_running(void) +{ + return periodic_timer != NULL; +} + +/** This function is called whenever the periodic_timer ticks. The function is + * responsible for moving the Tor process into an alertable state once a second + * and checking for whether our child processes have terminated since the last + * tick. */ +STATIC void +process_win32_timer_callback(periodic_timer_t *timer, void *data) +{ + tor_assert(timer == periodic_timer); + tor_assert(data == NULL); + + log_debug(LD_PROCESS, "Windows Process I/O timer ticked"); + const smartlist_t *processes = process_get_all_processes(); + + SMARTLIST_FOREACH(processes, process_t *, p, + process_win32_timer_test_process(p)); + + process_win32_trigger_completion_callbacks(); +} + +/** Test whether a given process is still alive. Notify the Process subsystem + * if our process have died. */ +STATIC void +process_win32_timer_test_process(process_t *process) +{ + tor_assert(process); + + /* No need to look at processes that don't claim they are running. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return; + + process_win32_t *win32_process = process_get_win32_process(process); + BOOL ret = FALSE; + DWORD exit_code = 0; + + /* We start by testing whether our process is still running. */ + ret = GetExitCodeProcess(win32_process->process_information.hProcess, + &exit_code); + + if (! ret) { + log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", + format_win32_error(GetLastError())); + return; + } + + /* Notify our process_t that our process have terminated. */ + if (exit_code != STILL_ACTIVE) + process_notify_event_exit(process, exit_code); +} + +/** Create a new overlapped named pipe. This function creates a new connected, + * named, pipe in *read_pipe and *write_pipe if the function is + * succesful. Returns true on sucess, false on failure. */ +STATIC bool +process_win32_create_pipe(HANDLE *read_pipe, + HANDLE *write_pipe, + SECURITY_ATTRIBUTES *attributes, + process_win32_pipe_type_t pipe_type) +{ + tor_assert(read_pipe); + tor_assert(write_pipe); + tor_assert(attributes); + + BOOL ret = FALSE; + + /* Buffer size. */ + const size_t size = 4096; + + /* Our additional read/write modes that depends on which pipe type we are + * creating. */ + DWORD read_mode = 0; + DWORD write_mode = 0; + + /* Generate the unique pipe name. */ + char pipe_name[MAX_PATH]; + static DWORD process_id = 0; + static DWORD counter = 0; + + if (process_id == 0) + process_id = GetCurrentProcessId(); + + tor_snprintf(pipe_name, sizeof(pipe_name), + "\\\\.\\Pipe\\Tor-Process-Pipe-%lu-%lu", + process_id, counter++); + + /* Only one of our handles can be overlapped. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + read_mode = FILE_FLAG_OVERLAPPED; + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + write_mode = FILE_FLAG_OVERLAPPED; + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + /* Setup our read and write handles. */ + HANDLE read_handle; + HANDLE write_handle; + + /* Create our named pipe. */ + read_handle = CreateNamedPipeA(pipe_name, + (PIPE_ACCESS_INBOUND|read_mode), + (PIPE_TYPE_BYTE|PIPE_WAIT), + 1, + size, + size, + 1000, + attributes); + + if (read_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateNamedPipeA() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + /* Create our file in the pipe namespace. */ + write_handle = CreateFileA(pipe_name, + GENERIC_WRITE, + 0, + attributes, + OPEN_EXISTING, + (FILE_ATTRIBUTE_NORMAL|write_mode), + NULL); + + if (write_handle == INVALID_HANDLE_VALUE) { + log_warn(LD_PROCESS, "CreateFileA() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + + return false; + } + + /* Set the inherit flag for our pipe. */ + switch (pipe_type) { + case PROCESS_WIN32_PIPE_TYPE_READER: + ret = SetHandleInformation(read_handle, HANDLE_FLAG_INHERIT, 0); + break; + case PROCESS_WIN32_PIPE_TYPE_WRITER: + ret = SetHandleInformation(write_handle, HANDLE_FLAG_INHERIT, 0); + break; + default: + /* LCOV_EXCL_START */ + tor_assert_nonfatal_unreached_once(); + /* LCOV_EXCL_STOP */ + } + + if (! ret) { + log_warn(LD_PROCESS, "SetHandleInformation() failed: %s", + format_win32_error(GetLastError())); + + CloseHandle(read_handle); + CloseHandle(write_handle); + + return false; + } + + /* Everything is good. */ + *read_pipe = read_handle; + *write_pipe = write_handle; + + return true; +} + +/** Cleanup a given handle. */ +STATIC void +process_win32_cleanup_handle(process_win32_handle_t *handle) +{ + tor_assert(handle); + +#if 0 + /* FIXME(ahf): My compiler does not set _WIN32_WINNT to a high enough value + * for this code to be available. Should we force it? CancelIoEx() is + * available from Windows 7 and above. If we decide to require this, we need + * to update the checks in all the three I/O completion callbacks to handle + * the ERROR_OPERATION_ABORTED as well as ERROR_BROKEN_PIPE. */ + +#if _WIN32_WINNT >= 0x0600 + /* This code is only supported from Windows 7 and onwards. */ + BOOL ret; + DWORD error_code; + + /* Cancel any pending I/O requests. */ + ret = CancelIoEx(handle->pipe, &handle->overlapped); + + if (! ret) { + error_code = GetLastError(); + + /* There was no pending I/O requests for our handle. */ + if (error_code != ERROR_NOT_FOUND) { + log_warn(LD_PROCESS, "CancelIoEx() failed: %s", + format_win32_error(error_code)); + } + } +#endif +#endif + + /* Close our handle. */ + if (handle->pipe != INVALID_HANDLE_VALUE) { + CloseHandle(handle->pipe); + handle->pipe = INVALID_HANDLE_VALUE; + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard output. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stdout_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* This happens when we have asked ReadFileEx() to read some data, but we + * then decided to call CloseHandle() on the HANDLE. This can happen if + * someone runs process_free() in the exit_callback of process_t, which means + * we cannot call process_get_win32_process() here. */ + if (error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "Process reported broken pipe on standard out"); + return; + } + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stdout_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stdout(process); + } +} + +/** This function is called when ReadFileEx() completes its background read + * from the child process's standard error. We notify the Process subsystem if + * there is data available for it to read from us. */ +STATIC VOID WINAPI +process_win32_stderr_read_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + /* This happens when we have asked ReadFileEx() to read some data, but we + * then decided to call CloseHandle() on the HANDLE. This can happen if + * someone runs process_free() in the exit_callback of process_t, which means + * we cannot call process_get_win32_process() here. */ + if (error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "Process reported broken pipe on standard error"); + return; + } + + /* Extract our process_t from the hEvent member of OVERLAPPED. */ + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + if (process_win32_handle_read_completion(&win32_process->stderr_handle, + error_code, + byte_count)) { + /* Schedule our next read. */ + process_notify_event_stderr(process); + } +} + +/** This function is called when WriteFileEx() completes its background write + * to the child process's standard input. We notify the Process subsystem that + * it can write data to us again. */ +STATIC VOID WINAPI +process_win32_stdin_write_done(DWORD error_code, + DWORD byte_count, + LPOVERLAPPED overlapped) +{ + tor_assert(overlapped); + tor_assert(overlapped->hEvent); + + (void)byte_count; + + /* This happens when we have asked WriteFileEx() to write some data, but we + * then decided to call CloseHandle() on the HANDLE. This can happen if + * someone runs process_free() in the exit_callback of process_t, which means + * we cannot call process_get_win32_process() here. */ + if (error_code == ERROR_BROKEN_PIPE) { + log_debug(LD_PROCESS, "Process reported broken pipe on standard input"); + return; + } + + process_t *process = (process_t *)overlapped->hEvent; + process_win32_t *win32_process = process_get_win32_process(process); + + /* Mark our handle as not having any outstanding I/O requests. */ + win32_process->stdin_handle.busy = false; + + /* Check if we have been asked to write to the handle that have been marked + * as having reached EOF. */ + if (BUG(win32_process->stdin_handle.reached_eof)) + return; + + if (error_code == 0) { + /** Our data have been succesfully written. Clear our state and schedule + * the next write. */ + win32_process->stdin_handle.data_available = 0; + memset(win32_process->stdin_handle.buffer, 0, + sizeof(win32_process->stdin_handle.buffer)); + + /* Schedule the next write. */ + process_notify_event_stdin(process); + } else if (error_code == ERROR_HANDLE_EOF) { + /* Our WriteFileEx() call was succesful, but we reached the end of our + * file. We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + win32_process->stdin_handle.reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from WriteFileEx(): %s", + format_win32_error(error_code)); + win32_process->stdin_handle.reached_eof = true; + } +} + +/** This function reads data from the given handle's internal buffer and + * moves it into the given buffer. Additionally, we start the next + * ReadFileEx() background operation with the given callback as + * completion callback. Returns the number of bytes written to the buffer. */ +STATIC int +process_win32_read_from_handle(process_win32_handle_t *handle, + buf_t *buffer, + LPOVERLAPPED_COMPLETION_ROUTINE callback) +{ + tor_assert(handle); + tor_assert(buffer); + tor_assert(callback); + + BOOL ret = FALSE; + int bytes_available = 0; + + /* We already have a request to read data that isn't complete yet. */ + if (BUG(handle->busy)) + return 0; + + /* Check if we have been asked to read from a handle that have already told + * us that we have reached the end of the file. */ + if (BUG(handle->reached_eof)) + return 0; + + /* This cast should be safe since our buffer can be at maximum up to + * BUFFER_SIZE in size. */ + bytes_available = (int)handle->data_available; + + if (handle->data_available > 0) { + /* Read data from our intermediate buffer into the process_t buffer. */ + buf_add(buffer, handle->buffer, handle->data_available); + + /* Reset our read state. */ + handle->data_available = 0; + memset(handle->buffer, 0, sizeof(handle->buffer)); + } + + /* Ask the Windows kernel to read data from our pipe into our buffer and call + * the callback function when it is done. */ + ret = ReadFileEx(handle->pipe, + handle->buffer, + sizeof(handle->buffer), + &handle->overlapped, + callback); + + if (! ret) { + log_warn(LD_PROCESS, "ReadFileEx() failed: %s", + format_win32_error(GetLastError())); + return bytes_available; + } + + /* We mark our handle as having a pending I/O request. */ + handle->busy = true; + + return bytes_available; +} + +/** This function checks the callback values from ReadFileEx() in + * error_code and byte_count if we have read data. Returns true + * iff our caller should request more data from ReadFileEx(). */ +STATIC bool +process_win32_handle_read_completion(process_win32_handle_t *handle, + DWORD error_code, + DWORD byte_count) +{ + tor_assert(handle); + + /* Mark our handle as not having any outstanding I/O requests. */ + handle->busy = false; + + if (error_code == 0) { + /* Our ReadFileEx() call was succesful and there is data for us. */ + + /* This cast should be safe since byte_count should never be larger than + * BUFFER_SIZE. */ + tor_assert(byte_count <= BUFFER_SIZE); + handle->data_available = (size_t)byte_count; + + /* Tell our caller to schedule the next read. */ + return true; + } else if (error_code == ERROR_HANDLE_EOF) { + /* Our ReadFileEx() call was succesful, but we reached the end of our file. + * We mark our handle as having reached EOF and returns. */ + tor_assert(byte_count == 0); + + handle->reached_eof = true; + } else { + /* An error happened: We warn the user and mark our handle as having + * reached EOF */ + log_warn(LD_PROCESS, + "Error in I/O completion routine from ReadFileEx(): %s", + format_win32_error(error_code)); + + handle->reached_eof = true; + } + + /* Our caller should NOT schedule the next read. */ + return false; +} + +#endif /* ! defined(_WIN32). */ diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h new file mode 100644 index 0000000000..8c3b80d341 --- /dev/null +++ b/src/lib/process/process_win32.h @@ -0,0 +1,90 @@ +/* Copyright (c) 2003-2004, Roger Dingledine + * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. + * Copyright (c) 2007-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file process_win32.h + * \brief Header for process_win32.c + **/ + +#ifndef TOR_PROCESS_WIN32_H +#define TOR_PROCESS_WIN32_H + +#ifdef _WIN32 + +#include "orconfig.h" +#include "lib/malloc/malloc.h" +#include "lib/evloop/compat_libevent.h" + +#include + +struct process_t; + +struct process_win32_t; +typedef struct process_win32_t process_win32_t; + +process_win32_t *process_win32_new(void); +void process_win32_free_(process_win32_t *win32_process); +#define process_win32_free(s) \ + FREE_AND_NULL(process_win32_t, process_win32_free_, (s)) + +void process_win32_init(void); +void process_win32_deinit(void); + +process_status_t process_win32_exec(struct process_t *process); + +int process_win32_write(struct process_t *process, buf_t *buffer); +int process_win32_read_stdout(struct process_t *process, buf_t *buffer); +int process_win32_read_stderr(struct process_t *process, buf_t *buffer); + +void process_win32_trigger_completion_callbacks(void); + +#ifdef PROCESS_WIN32_PRIVATE +/* Timer handling. */ +STATIC void process_win32_timer_start(void); +STATIC void process_win32_timer_stop(void); +STATIC bool process_win32_timer_running(void); +STATIC void process_win32_timer_callback(periodic_timer_t *, void *); +STATIC void process_win32_timer_test_process(process_t *); + +/* I/O pipe handling. */ +struct process_win32_handle_t; +typedef struct process_win32_handle_t process_win32_handle_t; + +typedef enum process_win32_pipe_type_t { + /** This pipe is used for reading. */ + PROCESS_WIN32_PIPE_TYPE_READER, + + /** This pipe is used for writing. */ + PROCESS_WIN32_PIPE_TYPE_WRITER +} process_win32_pipe_type_t; + +STATIC bool process_win32_create_pipe(HANDLE *, + HANDLE *, + SECURITY_ATTRIBUTES *, + process_win32_pipe_type_t); + +STATIC void process_win32_cleanup_handle(process_win32_handle_t *handle); + +STATIC VOID WINAPI process_win32_stdout_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stderr_read_done(DWORD, + DWORD, + LPOVERLAPPED); +STATIC VOID WINAPI process_win32_stdin_write_done(DWORD, + DWORD, + LPOVERLAPPED); + +STATIC int process_win32_read_from_handle(process_win32_handle_t *, + buf_t *, + LPOVERLAPPED_COMPLETION_ROUTINE); +STATIC bool process_win32_handle_read_completion(process_win32_handle_t *, + DWORD, + DWORD); +#endif /* defined(PROCESS_WIN32_PRIVATE). */ + +#endif /* ! defined(_WIN32). */ + +#endif /* defined(TOR_PROCESS_WIN32_H). */ diff --git a/src/test/test_process.c b/src/test/test_process.c index 816695ccab..85ee9691a4 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -14,6 +14,8 @@ #include "lib/process/process.h" #define PROCESS_UNIX_PRIVATE #include "lib/process/process_unix.h" +#define PROCESS_WIN32_PRIVATE +#include "lib/process/process_win32.h" static const char *stdout_read_buffer; static const char *stderr_read_buffer; @@ -553,7 +555,7 @@ test_unix(void *arg) #ifndef _WIN32 process_init(); - process_t *process = process_new(); + process_t *process = process_new(""); /* On Unix all processes should have a Unix process handle. */ tt_ptr_op(NULL, OP_NE, process_get_unix_process(process)); @@ -564,6 +566,24 @@ test_unix(void *arg) #endif } +static void +test_win32(void *arg) +{ + (void)arg; +#ifdef _WIN32 + process_init(); + + process_t *process = process_new(""); + + /* On Win32 all processes should have a Win32 process handle. */ + tt_ptr_op(NULL, OP_NE, process_get_win32_process(process)); + + done: + process_free(process); + process_free_all(); +#endif +} + struct testcase_t process_tests[] = { { "default_values", test_default_values, TT_FORK, NULL, NULL }, { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, @@ -575,5 +595,6 @@ struct testcase_t process_tests[] = { { "exit_simple", test_exit_simple, TT_FORK, NULL, NULL }, { "argv_simple", test_argv_simple, TT_FORK, NULL, NULL }, { "unix", test_unix, TT_FORK, NULL, NULL }, + { "win32", test_win32, TT_FORK, NULL, NULL }, END_OF_TESTCASES }; From 89393a77e5db804784d4f08ef67fd2831799d65b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 05:22:24 +0100 Subject: [PATCH 11/34] Add process_get_pid() to the Process subsystem. This patch adds support for getting the unique process identifier from a given process_t. This patch implements both support for both the Unix and Microsoft Windows backend. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 13 +++++++++++++ src/lib/process/process.h | 3 +++ src/lib/process/process_unix.c | 10 ++++++++++ src/lib/process/process_unix.h | 2 ++ src/lib/process/process_win32.c | 10 ++++++++++ src/lib/process/process_win32.h | 2 ++ src/test/test_process.c | 3 +++ 7 files changed, 43 insertions(+) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index d4237b2b1f..ab19378a93 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -255,6 +255,19 @@ process_exec(process_t *process) return status; } +/** Returns the unique process identifier for the given process. */ +process_pid_t +process_get_pid(process_t *process) +{ + tor_assert(process); + +#ifndef _WIN32 + return process_unix_get_pid(process); +#else + return process_win32_get_pid(process); +#endif +} + /** Set the callback function for output from the child process's standard out * handle. This function sets the callback function which is called every time * the child process have written output to its standard out file handle. diff --git a/src/lib/process/process.h b/src/lib/process/process.h index f759c71939..7fd6cf53d0 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -49,6 +49,7 @@ struct process_t; typedef struct process_t process_t; typedef uint64_t process_exit_code_t; +typedef uint64_t process_pid_t; typedef void (*process_read_callback_t)(process_t *, char *, @@ -66,6 +67,8 @@ void process_free_(process_t *process); process_status_t process_exec(process_t *process); +process_pid_t process_get_pid(process_t *process); + void process_set_stdout_read_callback(process_t *, process_read_callback_t); void process_set_stderr_read_callback(process_t *, diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index c3691f1851..fa03fdbbe4 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -356,6 +356,16 @@ process_unix_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Returns the unique process identifier for the given process. */ +process_pid_t +process_unix_get_pid(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + return (process_pid_t)unix_process->pid; +} + /** Write the given buffer as input to the given process's * standard input. Returns the number of bytes written. */ int diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index 5fc23bcf07..0474746b26 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -30,6 +30,8 @@ void process_unix_free_(process_unix_t *unix_process); process_status_t process_unix_exec(struct process_t *process); +process_pid_t process_unix_get_pid(struct process_t *process); + int process_unix_write(struct process_t *process, buf_t *buffer); int process_unix_read_stdout(struct process_t *process, buf_t *buffer); int process_unix_read_stderr(struct process_t *process, buf_t *buffer); diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index a019e0b4f3..3e97f37801 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -271,6 +271,16 @@ process_win32_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Returns the unique process identifier for the given process. */ +process_pid_t +process_win32_get_pid(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + return (process_pid_t)win32_process->process_information.dwProcessId; +} + /** Schedule an async write of the data found in buffer for the given * process. This function runs an async write operation of the content of * buffer, if we are not already waiting for a pending I/O request. Returns the diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index 8c3b80d341..dbd264104c 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -34,6 +34,8 @@ void process_win32_deinit(void); process_status_t process_win32_exec(struct process_t *process); +process_pid_t process_win32_get_pid(struct process_t *process); + int process_win32_write(struct process_t *process, buf_t *buffer); int process_win32_read_stdout(struct process_t *process, buf_t *buffer); int process_win32_read_stderr(struct process_t *process, buf_t *buffer); diff --git a/src/test/test_process.c b/src/test/test_process.c index 85ee9691a4..4f86e786cb 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -160,6 +160,9 @@ test_default_values(void *arg) tt_assert(smartlist_contains(process_get_all_processes(), process)); + /* Default PID is 0. */ + tt_int_op(0, OP_EQ, process_get_pid(process)); + /* Our arguments should be empty. */ tt_int_op(0, OP_EQ, smartlist_len(process_get_arguments(process))); From e982fb1dae6ff0888ae419246578048470dd65b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 17:38:40 +0100 Subject: [PATCH 12/34] Add documentation for the is_socket and error argument of read_to_chunk(). See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index 1b65819dbb..da7043d5cb 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -33,8 +33,10 @@ /** Read up to at_most bytes from the file descriptor fd into * chunk (which must be on buf). If we get an EOF, set - * *reached_eof to 1. Return -1 on error, 0 on eof or blocking, - * and the number of bytes read otherwise. */ + * *reached_eof to 1. Uses tor_socket_recv() iff is_socket + * is true, otherwise it uses read(). Return -1 on error (and sets + * *error to errno), 0 on eof or blocking, and the number of bytes read + * otherwise. */ static inline int read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, int *reached_eof, int *error, bool is_socket) From 338137221c8bd89f6d611c0cd3bf7b8a85d02517 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 18:14:03 +0100 Subject: [PATCH 13/34] Make sure we call process_notify_event_exit() as the last thing in different callbacks. This patch makes sure that we call process_notify_event_exit() after we have done any modifications we need to do to the state of a process_t. This allows application developers to call process_free() in the exit_callback of the process. See: https://bugs.torproject.org/28179 --- src/lib/process/process_unix.c | 10 +++++++--- src/lib/process/process_win32.c | 7 +++++-- 2 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index fa03fdbbe4..4f46bbd888 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -549,12 +549,16 @@ process_unix_waitpid_callback(int status, void *data) process_t *process = data; process_unix_t *unix_process = process_get_unix_process(process); - /* Notify our process. */ - process_notify_event_exit(process, status); - /* Remove our waitpid callback. */ clear_waitpid_callback(unix_process->waitpid); unix_process->waitpid = NULL; + + /* Notify our process. */ + process_notify_event_exit(process, status); + + /* Make sure you don't modify the process after we have called + * process_notify_event_exit() on it, to allow users to process_free() it in + * the exit callback. */ } /** This function sets the file descriptor in the handle as non-blocking diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 3e97f37801..7422493deb 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -445,12 +445,15 @@ process_win32_timer_callback(periodic_timer_t *timer, void *data) tor_assert(data == NULL); log_debug(LD_PROCESS, "Windows Process I/O timer ticked"); + + /* Move the process into an alertable state. */ + process_win32_trigger_completion_callbacks(); + + /* Check if our processes are still alive. */ const smartlist_t *processes = process_get_all_processes(); SMARTLIST_FOREACH(processes, process_t *, p, process_win32_timer_test_process(p)); - - process_win32_trigger_completion_callbacks(); } /** Test whether a given process is still alive. Notify the Process subsystem From 4f611a1df70d2c5e4cb6261f75c1b82c9ed04598 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 03:44:59 +0100 Subject: [PATCH 14/34] Add process_terminate(). This patch adds support for process termination to the Process subsystem. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 20 ++++++++++++++++++++ src/lib/process/process.h | 1 + src/lib/process/process_unix.c | 26 ++++++++++++++++++++++++++ src/lib/process/process_unix.h | 1 + src/lib/process/process_win32.c | 22 ++++++++++++++++++++++ src/lib/process/process_win32.h | 1 + 6 files changed, 71 insertions(+) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index ab19378a93..657b319f54 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -255,6 +255,26 @@ process_exec(process_t *process) return status; } +/** Terminate the given process. Returns true on success, + * otherwise false. */ +bool +process_terminate(process_t *process) +{ + tor_assert(process); + + /* Terminating a non-running process isn't going to work. */ + if (process_get_status(process) != PROCESS_STATUS_RUNNING) + return false; + + log_debug(LD_PROCESS, "Terminating process"); + +#ifndef _WIN32 + return process_unix_terminate(process); +#else + return process_win32_terminate(process); +#endif +} + /** Returns the unique process identifier for the given process. */ process_pid_t process_get_pid(process_t *process) diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 7fd6cf53d0..c6b733a065 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -66,6 +66,7 @@ void process_free_(process_t *process); #define process_free(s) FREE_AND_NULL(process_t, process_free_, (s)) process_status_t process_exec(process_t *process); +bool process_terminate(process_t *process); process_pid_t process_get_pid(process_t *process); diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 4f46bbd888..4a9aaa2edd 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -356,6 +356,32 @@ process_unix_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_unix_terminate(process_t *process) +{ + tor_assert(process); + + process_unix_t *unix_process = process_get_unix_process(process); + + /* All running processes should have a waitpid. */ + if (BUG(unix_process->waitpid == NULL)) + return false; + + /* Send a SIGTERM to our child process. */ + int ret; + + ret = kill(unix_process->pid, SIGTERM); + + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to terminate process: %s", + strerror(errno)); + return false; + } + + return ret == 0; +} + /** Returns the unique process identifier for the given process. */ process_pid_t process_unix_get_pid(process_t *process) diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index 0474746b26..e17c59ea81 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -29,6 +29,7 @@ void process_unix_free_(process_unix_t *unix_process); FREE_AND_NULL(process_unix_t, process_unix_free_, (s)) process_status_t process_unix_exec(struct process_t *process); +bool process_unix_terminate(struct process_t *process); process_pid_t process_unix_get_pid(struct process_t *process); diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 7422493deb..7b18903c70 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -271,6 +271,28 @@ process_win32_exec(process_t *process) return PROCESS_STATUS_RUNNING; } +/** Terminate the given process. Returns true on success, otherwise false. */ +bool +process_win32_terminate(process_t *process) +{ + tor_assert(process); + + process_win32_t *win32_process = process_get_win32_process(process); + + /* Terminate our process. */ + BOOL ret; + + ret = TerminateProcess(win32_process->process_information.hProcess, 0); + + if (! ret) { + log_warn(LD_PROCESS, "TerminateProcess() failed: %s", + format_win32_error(GetLastError())); + return false; + } + + return true; +} + /** Returns the unique process identifier for the given process. */ process_pid_t process_win32_get_pid(process_t *process) diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index dbd264104c..9a42e6c713 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -33,6 +33,7 @@ void process_win32_init(void); void process_win32_deinit(void); process_status_t process_win32_exec(struct process_t *process); +bool process_win32_terminate(struct process_t *process); process_pid_t process_win32_get_pid(struct process_t *process); From b0d268a82264f28f6b78f734d45a13e8272e70ea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 05:43:34 +0100 Subject: [PATCH 15/34] Add process_reset_environment() to the Process subsystem. This patch adds a new function that allows us to reset the environment of a given process_t with a list of key/value pairs. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 17 +++++++++++++++++ src/lib/process/process.h | 1 + 2 files changed, 18 insertions(+) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 657b319f54..915217e132 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -441,6 +441,23 @@ process_get_argv(const process_t *process) return argv; } +/** This function clears the internal environment and copies over every string + * from env as the new environment. */ +void +process_reset_environment(process_t *process, const smartlist_t *env) +{ + tor_assert(process); + tor_assert(env); + + /* Cleanup old environment. */ + SMARTLIST_FOREACH(process->environment, char *, x, tor_free(x)); + smartlist_free(process->environment); + process->environment = smartlist_new(); + + SMARTLIST_FOREACH(env, char *, x, + smartlist_add(process->environment, tor_strdup(x))); +} + /** Set the given key/value pair as environment variable in the * given process. */ void diff --git a/src/lib/process/process.h b/src/lib/process/process.h index c6b733a065..6092c2da7d 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -83,6 +83,7 @@ void process_append_argument(process_t *process, const char *argument); const smartlist_t *process_get_arguments(const process_t *process); char **process_get_argv(const process_t *process); +void process_reset_environment(process_t *process, const smartlist_t *env); void process_set_environment(process_t *process, const char *key, const char *value); From ad4cc89c5d1987cbcb231bf054433c7f05b83a95 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 22 Nov 2018 19:00:21 +0100 Subject: [PATCH 16/34] Add "PT" log domain. See: https://bugs.torproject.org/28179 --- doc/tor.1.txt | 4 ++-- src/lib/log/log.c | 2 +- src/lib/log/log.h | 4 +++- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/doc/tor.1.txt b/doc/tor.1.txt index 42fe8f1bcc..1984b05d64 100644 --- a/doc/tor.1.txt +++ b/doc/tor.1.txt @@ -669,8 +669,8 @@ GENERAL OPTIONS + The currently recognized domains are: general, crypto, net, config, fs, protocol, mm, http, app, control, circ, rend, bug, dir, dirserv, or, edge, - acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, and - process. + acct, hist, handshake, heartbeat, channel, sched, guard, consdiff, dos, + process, and pt. Domain names are case-insensitive. + + For example, "`Log [handshake]debug [~net,~mm]info notice stdout`" sends diff --git a/src/lib/log/log.c b/src/lib/log/log.c index 861de6e963..2c25ddf1aa 100644 --- a/src/lib/log/log.c +++ b/src/lib/log/log.c @@ -1268,7 +1268,7 @@ static const char *domain_list[] = { "GENERAL", "CRYPTO", "NET", "CONFIG", "FS", "PROTOCOL", "MM", "HTTP", "APP", "CONTROL", "CIRC", "REND", "BUG", "DIR", "DIRSERV", "OR", "EDGE", "ACCT", "HIST", "HANDSHAKE", "HEARTBEAT", "CHANNEL", - "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", NULL + "SCHED", "GUARD", "CONSDIFF", "DOS", "PROCESS", "PT", NULL }; /** Return a bitmask for the log domain for which domain is the name, diff --git a/src/lib/log/log.h b/src/lib/log/log.h index 1cd6087eb8..c8820ee037 100644 --- a/src/lib/log/log.h +++ b/src/lib/log/log.h @@ -109,8 +109,10 @@ #define LD_DOS (1u<<25) /** Processes */ #define LD_PROCESS (1u<<26) +/** Pluggable Transports. */ +#define LD_PT (1u<<27) /** Number of logging domains in the code. */ -#define N_LOGGING_DOMAINS 27 +#define N_LOGGING_DOMAINS 28 /** This log message is not safe to send to a callback-based logger * immediately. Used as a flag, not a log domain. */ From bfb94dd2ca8e04fb1fe8aba9ad48effbb8b70662 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 05:56:41 +0100 Subject: [PATCH 17/34] Use process_t for managed proxies. This patch makes the managed proxy subsystem use the process_t data structure such that we can get events from the PT process while Tor is running and not just when the PT process is being configured. See: https://bugs.torproject.org/28179 --- src/core/mainloop/mainloop.c | 4 - src/feature/client/transports.c | 174 ++++++++++++++++++-------------- src/feature/client/transports.h | 12 ++- src/test/test_pt.c | 66 ++++++------ 4 files changed, 137 insertions(+), 119 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index 2e2ae876d4..aaaa5009cb 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -1853,10 +1853,6 @@ second_elapsed_callback(time_t now, const or_options_t *options) run_connection_housekeeping(i, now); } - /* 11b. check pending unconfigured managed proxies */ - if (!net_is_disabled() && pt_proxies_configuration_pending()) - pt_configure_remaining_proxies(); - /* Run again in a second. */ return 1; } diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 85d4d2e080..f0400b713d 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -102,10 +102,11 @@ #include "feature/relay/ext_orport.h" #include "feature/control/control.h" -#include "lib/process/env.h" +#include "lib/process/process.h" #include "lib/process/subprocess.h" +#include "lib/process/env.h" -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp); static inline int proxy_configuration_finished(const managed_proxy_t *mp); @@ -490,8 +491,8 @@ proxy_prepare_for_restart(managed_proxy_t *mp) tor_assert(mp->conf_state == PT_PROTO_COMPLETED); /* destroy the process handle and terminate the process. */ - tor_process_handle_destroy(mp->process_handle, 1); - mp->process_handle = NULL; + process_set_data(mp->process, NULL); + process_terminate(mp->process); /* destroy all its registered transports, since we will no longer use them. */ @@ -520,34 +521,35 @@ proxy_prepare_for_restart(managed_proxy_t *mp) static int launch_managed_proxy(managed_proxy_t *mp) { - int retval; + tor_assert(mp); - process_environment_t *env = create_managed_proxy_environment(mp); + smartlist_t *env = create_managed_proxy_environment(mp); -#ifdef _WIN32 - /* Passing NULL as lpApplicationName makes Windows search for the .exe */ - retval = tor_spawn_background(NULL, - (const char **)mp->argv, - env, - &mp->process_handle); -#else /* !(defined(_WIN32)) */ - retval = tor_spawn_background(mp->argv[0], - (const char **)mp->argv, - env, - &mp->process_handle); -#endif /* defined(_WIN32) */ + /* Configure our process. */ + process_set_data(mp->process, mp); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_stderr_read_callback(mp->process, managed_proxy_stderr_callback); + process_set_exit_callback(mp->process, managed_proxy_exit_callback); + process_set_protocol(mp->process, PROCESS_PROTOCOL_LINE); + process_reset_environment(mp->process, env); - process_environment_free(env); + /* Cleanup our env. */ + SMARTLIST_FOREACH(env, char *, x, tor_free(x)); + smartlist_free(env); - if (retval == PROCESS_STATUS_ERROR) { - log_warn(LD_GENERAL, "Managed proxy at '%s' failed at launch.", + /* Skip the argv[0] as we get that from process_new(argv[0]). */ + for (int i = 1; mp->argv[i] != NULL; ++i) + process_append_argument(mp->process, mp->argv[i]); + + if (process_exec(mp->process) != PROCESS_STATUS_RUNNING) { + log_warn(LD_CONFIG, "Managed proxy at '%s' failed at launch.", mp->argv[0]); return -1; } - log_info(LD_CONFIG, "Managed proxy at '%s' has spawned with PID '%d'.", - mp->argv[0], tor_process_get_pid(mp->process_handle)); - + log_info(LD_CONFIG, + "Managed proxy at '%s' has spawned with PID '%" PRIu64 "'.", + mp->argv[0], process_get_pid(mp->process)); mp->conf_state = PT_PROTO_LAUNCHED; return 0; @@ -615,10 +617,6 @@ pt_configure_remaining_proxies(void) STATIC int configure_proxy(managed_proxy_t *mp) { - int configuration_finished = 0; - smartlist_t *proxy_output = NULL; - enum stream_status stream_status = 0; - /* if we haven't launched the proxy yet, do it now */ if (mp->conf_state == PT_PROTO_INFANT) { if (launch_managed_proxy(mp) < 0) { /* launch fail */ @@ -629,45 +627,8 @@ configure_proxy(managed_proxy_t *mp) } tor_assert(mp->conf_state != PT_PROTO_INFANT); - tor_assert(mp->process_handle); - - proxy_output = - tor_get_lines_from_handle(tor_process_get_stdout_pipe(mp->process_handle), - &stream_status); - if (!proxy_output) { /* failed to get input from proxy */ - if (stream_status != IO_STREAM_EAGAIN) { /* bad stream status! */ - mp->conf_state = PT_PROTO_BROKEN; - log_warn(LD_GENERAL, "The communication stream of managed proxy '%s' " - "is '%s'. Most probably the managed proxy stopped running. " - "This might be a bug of the managed proxy, a bug of Tor, or " - "a misconfiguration. Please enable logging on your managed " - "proxy and check the logs for errors.", - mp->argv[0], stream_status_to_string(stream_status)); - } - - goto done; - } - - /* Handle lines. */ - SMARTLIST_FOREACH_BEGIN(proxy_output, const char *, line) { - handle_proxy_line(line, mp); - if (proxy_configuration_finished(mp)) - goto done; - } SMARTLIST_FOREACH_END(line); - - done: - /* if the proxy finished configuring, exit the loop. */ - if (proxy_configuration_finished(mp)) { - handle_finished_proxy(mp); - configuration_finished = 1; - } - - if (proxy_output) { - SMARTLIST_FOREACH(proxy_output, char *, cp, tor_free(cp)); - smartlist_free(proxy_output); - } - - return configuration_finished; + tor_assert(mp->process); + return mp->conf_state == PT_PROTO_COMPLETED; } /** Register server managed proxy mp transports to state */ @@ -748,8 +709,11 @@ managed_proxy_destroy(managed_proxy_t *mp, /* free the outgoing proxy URI */ tor_free(mp->proxy_uri); - tor_process_handle_destroy(mp->process_handle, also_terminate_process); - mp->process_handle = NULL; + /* do we want to terminate our process if it's still running? */ + if (also_terminate_process && mp->process) + process_terminate(mp->process); + + process_free(mp->process); tor_free(mp); } @@ -1257,7 +1221,7 @@ get_bindaddr_for_server_proxy(const managed_proxy_t *mp) /** Return a newly allocated process_environment_t * for mp's * process. */ -static process_environment_t * +static smartlist_t * create_managed_proxy_environment(const managed_proxy_t *mp) { const or_options_t *options = get_options(); @@ -1272,8 +1236,6 @@ create_managed_proxy_environment(const managed_proxy_t *mp) /* The final environment to be passed to mp. */ smartlist_t *merged_env_vars = get_current_process_environment_variables(); - process_environment_t *env; - { char *state_tmp = get_datadir_fname("pt_state/"); /* XXX temp */ smartlist_add_asprintf(envs, "TOR_PT_STATE_LOCATION=%s", state_tmp); @@ -1366,14 +1328,9 @@ create_managed_proxy_environment(const managed_proxy_t *mp) tor_free_, 1); } SMARTLIST_FOREACH_END(env_var); - env = process_environment_make(merged_env_vars); - smartlist_free(envs); - SMARTLIST_FOREACH(merged_env_vars, void *, x, tor_free(x)); - smartlist_free(merged_env_vars); - - return env; + return merged_env_vars; } /** Create and return a new managed proxy for transport using @@ -1392,6 +1349,7 @@ managed_proxy_create(const smartlist_t *with_transport_list, mp->argv = proxy_argv; mp->transports = smartlist_new(); mp->proxy_uri = get_pt_proxy_uri(); + mp->process = process_new(proxy_argv[0]); mp->transports_to_launch = smartlist_new(); SMARTLIST_FOREACH(with_transport_list, const char *, transport, @@ -1736,3 +1694,63 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) return new_string; } + +/** Callback function that is called when our PT process have data on its + * stdout. Our process can be found in process, the data can be found in + * line and the length of our line is given in size. */ +STATIC void +managed_proxy_stdout_callback(process_t *process, char *line, size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + handle_proxy_line(line, mp); + + if (proxy_configuration_finished(mp)) { + handle_finished_proxy(mp); + tor_assert(mp->conf_state == PT_PROTO_COMPLETED); + } +} + +/** Callback function that is called when our PT process have data on its + * stderr. Our process can be found in process, the data can be found in + * line and the length of our line is given in size. */ +STATIC void +managed_proxy_stderr_callback(process_t *process, char *line, size_t size) +{ + tor_assert(process); + tor_assert(line); + + (void)size; + + managed_proxy_t *mp = process_get_data(process); + + log_warn(LD_PT, "Managed proxy at '%s' reported: %s", mp->argv[0], line); +} + +/** Callback function that is called when our PT process terminates. The + * process exit code can be found in exit_code and our process can be + * found in process. */ +STATIC void +managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tor_assert(process); + + log_warn(LD_PT, + "Pluggable Transport process terminated with status code %" PRIu64, + exit_code); + + /* We detach ourself from the MP (if we are attached) and free ourself. */ + managed_proxy_t *mp = process_get_data(process); + + if (BUG(mp != NULL)) { + mp->process = NULL; + process_set_data(process, NULL); + } + + process_free(process); +} diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index b80875c95c..59df637d81 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -11,6 +11,8 @@ #ifndef TOR_TRANSPORTS_H #define TOR_TRANSPORTS_H +#include "lib/process/process.h" + /** Represents a pluggable transport used by a bridge. */ typedef struct transport_t { /** SOCKS version: One of PROXY_SOCKS4, PROXY_SOCKS5. */ @@ -81,7 +83,7 @@ enum pt_proto_state { PT_PROTO_FAILED_LAUNCH /* failed while launching */ }; -struct process_handle_t; +struct process_t; /** Structure containing information of a managed proxy. */ typedef struct { @@ -94,8 +96,8 @@ typedef struct { int is_server; /* is it a server proxy? */ - /* A pointer to the process handle of this managed proxy. */ - struct process_handle_t *process_handle; + /* A pointer to the process of this managed proxy. */ + struct process_t *process; /** Boolean: We are re-parsing our config, and we are going to * remove this managed proxy if we don't find it any transport @@ -140,6 +142,10 @@ STATIC char* get_pt_proxy_uri(void); STATIC void free_execve_args(char **arg); +STATIC void managed_proxy_stdout_callback(process_t *, char *, size_t); +STATIC void managed_proxy_stderr_callback(process_t *, char *, size_t); +STATIC void managed_proxy_exit_callback(process_t *, process_exit_code_t); + #endif /* defined(PT_PRIVATE) */ #endif /* !defined(TOR_TRANSPORTS_H) */ diff --git a/src/test/test_pt.c b/src/test/test_pt.c index d0160d1148..2501b867bb 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -9,6 +9,7 @@ #define STATEFILE_PRIVATE #define CONTROL_PRIVATE #define SUBPROCESS_PRIVATE +#define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" #include "app/config/confparse.h" @@ -20,6 +21,7 @@ #include "lib/process/subprocess.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" +#include "lib/process/process.h" #include "app/config/or_state_st.h" @@ -151,6 +153,8 @@ test_pt_get_transport_options(void *arg) config_line_t *cl = NULL; (void)arg; + process_init(); + execve_args = tor_malloc(sizeof(char*)*2); execve_args[0] = tor_strdup("cheeseshop"); execve_args[1] = NULL; @@ -190,6 +194,7 @@ test_pt_get_transport_options(void *arg) config_free_lines(cl); managed_proxy_destroy(mp, 0); smartlist_free(transport_list); + process_free_all(); } static void @@ -253,6 +258,8 @@ test_pt_get_extrainfo_string(void *arg) char *s = NULL; (void) arg; + process_init(); + argv1 = tor_malloc_zero(sizeof(char*)*3); argv1[0] = tor_strdup("ewige"); argv1[1] = tor_strdup("Blumenkraft"); @@ -286,43 +293,25 @@ test_pt_get_extrainfo_string(void *arg) smartlist_free(t1); smartlist_free(t2); tor_free(s); + process_free_all(); } -#ifdef _WIN32 -#define STDIN_HANDLE HANDLE* -#else -#define STDIN_HANDLE int -#endif - -static smartlist_t * -tor_get_lines_from_handle_replacement(STDIN_HANDLE handle, - enum stream_status *stream_status_out) +static int +process_read_stdout_replacement(process_t *process, buf_t *buffer) { + (void)process; static int times_called = 0; - smartlist_t *retval_sl = smartlist_new(); - - (void) handle; - (void) stream_status_out; /* Generate some dummy CMETHOD lines the first 5 times. The 6th time, send 'CMETHODS DONE' to finish configuring the proxy. */ if (times_called++ != 5) { - smartlist_add_asprintf(retval_sl, "SMETHOD mock%d 127.0.0.1:555%d", + buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n", times_called, times_called); } else { - smartlist_add_strdup(retval_sl, "SMETHODS DONE"); + buf_add_string(buffer, "SMETHODS DONE\n"); } - return retval_sl; -} - -/* NOP mock */ -static void -tor_process_handle_destroy_replacement(process_handle_t *process_handle, - int also_terminate_process) -{ - (void) process_handle; - (void) also_terminate_process; + return (int)buf_datalen(buffer); } static or_state_t *dummy_state = NULL; @@ -355,12 +344,11 @@ test_pt_configure_proxy(void *arg) managed_proxy_t *mp = NULL; (void) arg; + process_init(); + dummy_state = tor_malloc_zero(sizeof(or_state_t)); - MOCK(tor_get_lines_from_handle, - tor_get_lines_from_handle_replacement); - MOCK(tor_process_handle_destroy, - tor_process_handle_destroy_replacement); + MOCK(process_read_stdout, process_read_stdout_replacement); MOCK(get_or_state, get_or_state_replacement); MOCK(queue_control_event_string, @@ -372,24 +360,34 @@ test_pt_configure_proxy(void *arg) mp->conf_state = PT_PROTO_ACCEPTING_METHODS; mp->transports = smartlist_new(); mp->transports_to_launch = smartlist_new(); - mp->process_handle = tor_malloc_zero(sizeof(process_handle_t)); mp->argv = tor_malloc_zero(sizeof(char*)*2); mp->argv[0] = tor_strdup(""); mp->is_server = 1; + /* Configure the process. */ + mp->process = process_new(""); + process_set_stdout_read_callback(mp->process, managed_proxy_stdout_callback); + process_set_data(mp->process, mp); + /* Test the return value of configure_proxy() by calling it some times while it is uninitialized and then finally finalizing its configuration. */ for (i = 0 ; i < 5 ; i++) { + /* force a read from our mocked stdout reader. */ + process_notify_event_stdout(mp->process); + /* try to configure our proxy. */ retval = configure_proxy(mp); /* retval should be zero because proxy hasn't finished configuring yet */ tt_int_op(retval, OP_EQ, 0); /* check the number of registered transports */ - tt_assert(smartlist_len(mp->transports) == i+1); + tt_int_op(smartlist_len(mp->transports), OP_EQ, i+1); /* check that the mp is still waiting for transports */ tt_assert(mp->conf_state == PT_PROTO_ACCEPTING_METHODS); } + /* Get the SMETHOD DONE written to the process. */ + process_notify_event_stdout(mp->process); + /* this last configure_proxy() should finalize the proxy configuration. */ retval = configure_proxy(mp); /* retval should be 1 since the proxy finished configuring */ @@ -435,8 +433,7 @@ test_pt_configure_proxy(void *arg) done: or_state_free(dummy_state); - UNMOCK(tor_get_lines_from_handle); - UNMOCK(tor_process_handle_destroy); + UNMOCK(process_read_stdout); UNMOCK(get_or_state); UNMOCK(queue_control_event_string); if (controlevent_msgs) { @@ -449,10 +446,11 @@ test_pt_configure_proxy(void *arg) smartlist_free(mp->transports); } smartlist_free(mp->transports_to_launch); - tor_free(mp->process_handle); + process_free(mp->process); tor_free(mp->argv[0]); tor_free(mp->argv); tor_free(mp); + process_free_all(); } /* Test the get_pt_proxy_uri() function. */ From e3ceaebba25127a1d4a3da16e9128ab52491c080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 23 Nov 2018 22:50:55 +0100 Subject: [PATCH 18/34] Add support for logging messages from pluggable transports. This patch adds support for the "LOG" protocol message from a pluggable transport. This allows pluggable transport developers to relay log messages from their binary to Tor, which will both emit them as log messages from the Tor process itself, but also pass them on via the control port. See: https://bugs.torproject.org/28180 See: https://bugs.torproject.org/28181 See: https://bugs.torproject.org/28182 --- src/feature/client/transports.c | 42 +++++++++++++++++++++++++++++++++ src/feature/client/transports.h | 1 + src/feature/control/control.c | 11 +++++++++ src/feature/control/control.h | 5 +++- src/test/test_pt.c | 19 +++++++++++++-- 5 files changed, 75 insertions(+), 3 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index f0400b713d..dc76e9c245 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -128,6 +128,7 @@ static void parse_method_error(const char *line, int is_server_method); #define PROTO_SMETHODS_DONE "SMETHODS DONE" #define PROTO_PROXY_DONE "PROXY DONE" #define PROTO_PROXY_ERROR "PROXY-ERROR" +#define PROTO_LOG "LOG" /** The first and only supported - at the moment - configuration protocol version. */ @@ -909,6 +910,9 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; + } else if (!strcmpstart(line, PROTO_LOG)) { + parse_log_line(line); + return; } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { /* managed proxy launch failed: parse error message to learn why. */ int retval, child_state, saved_errno; @@ -1146,6 +1150,44 @@ parse_proxy_error(const char *line) line+strlen(PROTO_PROXY_ERROR)+1); } +/** Parses a LOG line and emit log events accordingly. */ +STATIC void +parse_log_line(const char *line) +{ + smartlist_t *items = smartlist_new(); + + if (strlen(line) < (strlen(PROTO_LOG) + 1)) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with missing arguments.", PROTO_LOG); + goto done; + } + + const char *arguments = line + strlen(PROTO_LOG) + 1; + + /* The format is 'LOG '. We accept empty messages. */ + smartlist_split_string(items, arguments, NULL, + SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); + + if (smartlist_len(items) < 2) { + log_warn(LD_PT, "Managed proxy sent us a %s line " + "with too few arguments.", PROTO_LOG); + goto done; + } + + const char *transport_name = smartlist_get(items, 0); + const char *message = smartlist_get(items, 1); + + log_info(LD_PT, "Managed proxy transport \"%s\" says: %s", + transport_name, message); + + /* Emit control port event. */ + control_event_transport_log(transport_name, message); + + done: + SMARTLIST_FOREACH(items, char *, s, tor_free(s)); + smartlist_free(items); +} + /** Return a newly allocated string that tor should place in * TOR_PT_SERVER_TRANSPORT_OPTIONS while configuring the server * manged proxy in mp. Return NULL if no such options are found. */ diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index 59df637d81..fbb720aac6 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -128,6 +128,7 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); +STATIC void parse_log_line(const char *line); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, diff --git a/src/feature/control/control.c b/src/feature/control/control.c index 94679dfd22..b6505a85d6 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -7395,6 +7395,17 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } +/** A pluggable transport called transport_name has emitted a log + * message found in message. */ +void +control_event_transport_log(const char *transport_name, const char *message) +{ + send_control_event(EVENT_TRANSPORT_LOG, + "650 TRANSPORT_LOG %s %s\r\n", + transport_name, + message); +} + /** Convert rendezvous auth type to string for HS_DESC control events */ const char * diff --git a/src/feature/control/control.h b/src/feature/control/control.h index cd5402d455..eb2b5676ea 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -205,6 +205,8 @@ void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); +void control_event_transport_log(const char *transport_name, + const char *message); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -293,7 +295,8 @@ void control_free_all(void); #define EVENT_HS_DESC 0x0021 #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_MAX_ 0x0023 +#define EVENT_TRANSPORT_LOG 0x0024 +#define EVENT_MAX_ 0x0024 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ #define EVENT_CAPACITY_ 0x0040 diff --git a/src/test/test_pt.c b/src/test/test_pt.c index 2501b867bb..c980276b14 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -304,11 +304,16 @@ process_read_stdout_replacement(process_t *process, buf_t *buffer) /* Generate some dummy CMETHOD lines the first 5 times. The 6th time, send 'CMETHODS DONE' to finish configuring the proxy. */ - if (times_called++ != 5) { + times_called++; + + if (times_called <= 5) { buf_add_printf(buffer, "SMETHOD mock%d 127.0.0.1:555%d\n", times_called, times_called); - } else { + } else if (times_called <= 6) { buf_add_string(buffer, "SMETHODS DONE\n"); + } else if (times_called <= 7) { + buf_add_string(buffer, "LOG mock3 Oh noes, something bad happened. " + "What do we do!?\n"); } return (int)buf_datalen(buffer); @@ -410,6 +415,16 @@ test_pt_configure_proxy(void *arg) tt_str_op(smartlist_get(controlevent_msgs, 4), OP_EQ, "650 TRANSPORT_LAUNCHED server mock5 127.0.0.1 5555\r\n"); + /* Get the log message out. */ + process_notify_event_stdout(mp->process); + + tt_int_op(controlevent_n, OP_EQ, 6); + tt_int_op(controlevent_event, OP_EQ, EVENT_TRANSPORT_LOG); + tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 6); + tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, + "650 TRANSPORT_LOG mock3 Oh noes, something bad happened. " + "What do we do!?\r\n"); + { /* check that the transport info were saved properly in the tor state */ config_line_t *transport_in_state = NULL; smartlist_t *transport_info_sl = smartlist_new(); From 9b6a10a26f25e0da07e62bd5617b519b0952b40a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 02:18:04 +0100 Subject: [PATCH 19/34] Add slow test for process_t for main loop interaction. This patch adds test cases for process_t which uses Tor's main loop. This allows us to test that the callbacks are actually invoked by the main loop when we expect them. See: https://bugs.torproject.org/28179 --- .gitignore | 2 + src/test/include.am | 2 + src/test/test-process.c | 85 +++++++++ src/test/test.h | 1 + src/test/test_process_slow.c | 330 +++++++++++++++++++++++++++++++++++ src/test/test_slow.c | 1 + 6 files changed, 421 insertions(+) create mode 100644 src/test/test-process.c create mode 100644 src/test/test_process_slow.c diff --git a/.gitignore b/.gitignore index ee2de376a6..df8db1113a 100644 --- a/.gitignore +++ b/.gitignore @@ -240,6 +240,7 @@ uptime-*.json /src/test/test-slow /src/test/test-bt-cl /src/test/test-child +/src/test/test-process /src/test/test-memwipe /src/test/test-ntor-cl /src/test/test-hs-ntor-cl @@ -250,6 +251,7 @@ uptime-*.json /src/test/test-slow.exe /src/test/test-bt-cl.exe /src/test/test-child.exe +/src/test/test-process.exe /src/test/test-ntor-cl.exe /src/test/test-hs-ntor-cl.exe /src/test/test-memwipe.exe diff --git a/src/test/include.am b/src/test/include.am index 482897c7a5..9df2fd481d 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -66,6 +66,7 @@ noinst_PROGRAMS+= \ src/test/test-slow \ src/test/test-memwipe \ src/test/test-child \ + src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ src/test/test-timers @@ -202,6 +203,7 @@ if UNITTESTS_ENABLED src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ + src/test/test_process_slow.c \ src/test/test_util_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ diff --git a/src/test/test-process.c b/src/test/test-process.c new file mode 100644 index 0000000000..ec1b395002 --- /dev/null +++ b/src/test/test-process.c @@ -0,0 +1,85 @@ +/* Copyright (c) 2011-2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +#include "orconfig.h" +#include +#ifdef _WIN32 +#define WINDOWS_LEAN_AND_MEAN +#include +#else +#include +#endif /* defined(_WIN32) */ +#include +#include + +#ifdef _WIN32 +#define SLEEP(sec) Sleep((sec)*1000) +#else +#define SLEEP(sec) sleep(sec) +#endif + +/* Trivial test program to test process_t. */ +int +main(int argc, char **argv) +{ + /* Does our process get the right arguments? */ + for (int i = 0; i < argc; ++i) { + fprintf(stdout, "argv[%d] = '%s'\n", i, argv[i]); + fflush(stdout); + } + + /* Make sure our process got our environment variable. */ + fprintf(stdout, "Environment variable TOR_TEST_ENV = '%s'\n", + getenv("TOR_TEST_ENV")); + fflush(stdout); + + /* Test line handling on stdout and stderr. */ + fprintf(stdout, "Output on stdout\nThis is a new line\n"); + fflush(stdout); + + fprintf(stderr, "Output on stderr\nThis is a new line\n"); + fflush(stderr); + + fprintf(stdout, "Partial line on stdout ..."); + fflush(stdout); + + fprintf(stderr, "Partial line on stderr ..."); + fflush(stderr); + + SLEEP(2); + + fprintf(stdout, "end of partial line on stdout\n"); + fflush(stdout); + fprintf(stderr, "end of partial line on stderr\n"); + fflush(stderr); + + /* Echo input from stdin. */ + char buffer[1024]; + + int count = 0; + + while (fgets(buffer, sizeof(buffer), stdin)) { + /* Strip the newline. */ + size_t size = strlen(buffer); + + if (size >= 1 && buffer[size - 1] == '\n') { + buffer[size - 1] = '\0'; + --size; + } + + if (size >= 1 && buffer[size - 1] == '\r') { + buffer[size - 1] = '\0'; + --size; + } + + fprintf(stdout, "Read line from stdin: '%s'\n", buffer); + fflush(stdout); + + if (++count == 3) + break; + } + + fprintf(stdout, "We are done for here, thank you!\n"); + + return 0; +} diff --git a/src/test/test.h b/src/test/test.h index 7c17389776..9f72623968 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -270,6 +270,7 @@ extern struct testcase_t voting_schedule_tests[]; extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; +extern struct testcase_t slow_process_tests[]; extern struct testcase_t slow_util_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c new file mode 100644 index 0000000000..b4c5a2d5e2 --- /dev/null +++ b/src/test/test_process_slow.c @@ -0,0 +1,330 @@ +/* Copyright (c) 2018, The Tor Project, Inc. */ +/* See LICENSE for licensing information */ + +/** + * \file test_process_slow.c + * \brief Slow test cases for the Process API. + */ + +#include "orconfig.h" +#include "core/or/or.h" +#include "core/mainloop/mainloop.h" +#include "lib/evloop/compat_libevent.h" +#include "lib/process/process.h" +#include "lib/process/waitpid.h" +#include "test/test.h" + +#ifndef BUILDDIR +#define BUILDDIR "." +#endif + +#ifdef _WIN32 +#define TEST_PROCESS "test-process.exe" +#else +#define TEST_PROCESS BUILDDIR "/src/test/test-process" +#endif /* defined(_WIN32) */ + +/** Timer that ticks once a second and stop the event loop after 5 ticks. */ +static periodic_timer_t *main_loop_timeout_timer; + +/** How many times have our timer ticked? */ +static int timer_tick_count; + +struct process_data_t { + smartlist_t *stdout_data; + smartlist_t *stderr_data; + smartlist_t *stdin_data; + process_exit_code_t exit_code; +}; + +typedef struct process_data_t process_data_t; + +static process_data_t * +process_data_new(void) +{ + process_data_t *process_data = tor_malloc_zero(sizeof(process_data_t)); + process_data->stdout_data = smartlist_new(); + process_data->stderr_data = smartlist_new(); + process_data->stdin_data = smartlist_new(); + return process_data; +} + +static void +process_data_free(process_data_t *process_data) +{ + if (process_data == NULL) + return; + + SMARTLIST_FOREACH(process_data->stdout_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stderr_data, char *, x, tor_free(x)); + SMARTLIST_FOREACH(process_data->stdin_data, char *, x, tor_free(x)); + + smartlist_free(process_data->stdout_data); + smartlist_free(process_data->stderr_data); + smartlist_free(process_data->stdin_data); + tor_free(process_data); +} + +static void +process_stdout_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stdout_data, tor_strdup(data)); + + done: + return; +} + +static void +process_stderr_callback(process_t *process, char *data, size_t size) +{ + tt_ptr_op(process, OP_NE, NULL); + tt_ptr_op(data, OP_NE, NULL); + tt_int_op(strlen(data), OP_EQ, size); + + process_data_t *process_data = process_get_data(process); + smartlist_add(process_data->stderr_data, tor_strdup(data)); + + done: + return; +} + +static void +process_exit_callback(process_t *process, process_exit_code_t exit_code) +{ + tt_ptr_op(process, OP_NE, NULL); + + process_data_t *process_data = process_get_data(process); + process_data->exit_code = exit_code; + + /* Our process died. Let's check the values it returned. */ + tor_shutdown_event_loop_and_exit(0); + + done: + return; +} + +#ifdef _WIN32 +static const char * +get_win32_test_binary_path(void) +{ + static char buffer[MAX_PATH]; + + /* Get the absolute path of our binary: \path\to\test-slow.exe. */ + GetModuleFileNameA(GetModuleHandle(0), buffer, sizeof(buffer)); + + /* Find our process name. */ + char *offset = strstr(buffer, "test-slow.exe"); + tt_ptr_op(offset, OP_NE, NULL); + + /* Change test-slow.exe to test-process.exe. */ + memcpy(offset, TEST_PROCESS, strlen(TEST_PROCESS)); + + return buffer; + done: + return NULL; +} +#endif + +static void +main_loop_timeout_cb(periodic_timer_t *timer, void *data) +{ + /* Sanity check. */ + tt_ptr_op(timer, OP_EQ, main_loop_timeout_timer); + tt_ptr_op(data, OP_EQ, NULL); + + /* Have we been called 10 times we exit. */ + timer_tick_count++; + + tt_int_op(timer_tick_count, OP_LT, 10); + +#ifndef _WIN32 + /* Call waitpid callbacks. */ + notify_pending_waitpid_callbacks(); +#endif + + return; + done: + /* Exit with an error. */ + tor_shutdown_event_loop_and_exit(-1); +} + +static void +run_main_loop(void) +{ + int ret; + + /* Wake up after 1 seconds. */ + static const struct timeval interval = {1, 0}; + + timer_tick_count = 0; + main_loop_timeout_timer = periodic_timer_new(tor_libevent_get_base(), + &interval, + main_loop_timeout_cb, + NULL); + + /* Run our main loop. */ + ret = do_main_loop(); + + /* Clean up our main loop timeout timer. */ + tt_int_op(ret, OP_EQ, 0); + + done: + periodic_timer_free(main_loop_timeout_timer); +} + +static void +test_callbacks(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Initialize Process subsystem. */ + process_init(); + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_stdout_read_callback(process, process_stdout_callback); + process_set_stderr_read_callback(process, process_stderr_callback); + process_set_exit_callback(process, process_exit_callback); + + /* Set environment variable. */ + process_set_environment(process, "TOR_TEST_ENV", "Hello, from Tor!"); + + /* Add some arguments. */ + process_append_argument(process, "This is the first one"); + process_append_argument(process, "Second one"); + process_append_argument(process, "Third: Foo bar baz"); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Write some lines to stdin. */ + process_printf(process, "Hi process!\r\n"); + process_printf(process, "Can you read more than one line?\n"); + process_printf(process, "Can you read partial ..."); + process_printf(process, " lines?\r\n"); + + /* Start our main loop. */ + run_main_loop(); + + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + + /* We returned. Let's see what our event loop said. */ + tt_int_op(smartlist_len(process_data->stdout_data), OP_EQ, 12); + tt_int_op(smartlist_len(process_data->stderr_data), OP_EQ, 3); + tt_int_op(process_data->exit_code, OP_EQ, 0); + + /* Check stdout output. */ + char argv0_expected[256]; + tor_snprintf(argv0_expected, sizeof(argv0_expected), + "argv[0] = '%s'", filename); + + tt_str_op(smartlist_get(process_data->stdout_data, 0), OP_EQ, + argv0_expected); + tt_str_op(smartlist_get(process_data->stdout_data, 1), OP_EQ, + "argv[1] = 'This is the first one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 2), OP_EQ, + "argv[2] = 'Second one'"); + tt_str_op(smartlist_get(process_data->stdout_data, 3), OP_EQ, + "argv[3] = 'Third: Foo bar baz'"); + tt_str_op(smartlist_get(process_data->stdout_data, 4), OP_EQ, + "Environment variable TOR_TEST_ENV = 'Hello, from Tor!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 5), OP_EQ, + "Output on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 6), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stdout_data, 7), OP_EQ, + "Partial line on stdout ...end of partial line on stdout"); + tt_str_op(smartlist_get(process_data->stdout_data, 8), OP_EQ, + "Read line from stdin: 'Hi process!'"); + tt_str_op(smartlist_get(process_data->stdout_data, 9), OP_EQ, + "Read line from stdin: 'Can you read more than one line?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 10), OP_EQ, + "Read line from stdin: 'Can you read partial ... lines?'"); + tt_str_op(smartlist_get(process_data->stdout_data, 11), OP_EQ, + "We are done for here, thank you!"); + + /* Check stderr output. */ + tt_str_op(smartlist_get(process_data->stderr_data, 0), OP_EQ, + "Output on stderr"); + tt_str_op(smartlist_get(process_data->stderr_data, 1), OP_EQ, + "This is a new line"); + tt_str_op(smartlist_get(process_data->stderr_data, 2), OP_EQ, + "Partial line on stderr ...end of partial line on stderr"); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); +} + +static void +test_callbacks_terminate(void *arg) +{ + (void)arg; + const char *filename = NULL; + +#ifdef _WIN32 + filename = get_win32_test_binary_path(); +#else + filename = TEST_PROCESS; +#endif + + /* Initialize Process subsystem. */ + process_init(); + + /* Process callback data. */ + process_data_t *process_data = process_data_new(); + + /* Setup our process. */ + process_t *process = process_new(filename); + process_set_data(process, process_data); + process_set_exit_callback(process, process_exit_callback); + + /* Run our process. */ + process_status_t status; + + status = process_exec(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); + + /* Zap our process. */ + process_terminate(process); + + /* Start our main loop. */ + run_main_loop(); + + /* Check if our process is still running? */ + status = process_get_status(process); + tt_int_op(status, OP_EQ, PROCESS_STATUS_NOT_RUNNING); + + done: + process_data_free(process_data); + process_free(process); + process_free_all(); +} + +struct testcase_t slow_process_tests[] = { + { "callbacks", test_callbacks, TT_FORK, NULL, NULL }, + { "callbacks_terminate", test_callbacks_terminate, TT_FORK, NULL, NULL }, + END_OF_TESTCASES +}; diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 0b665363ab..0c66eff930 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -20,6 +20,7 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, + { "slow/process/", slow_process_tests }, { "slow/util/", slow_util_tests }, END_OF_GROUPS }; From 289ed0849d9b473cd0fbafada30c03638d73e541 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 04:59:12 +0100 Subject: [PATCH 20/34] Add tests for process environment functionality of process_t. This patch adds tests for the process_environment_t interaction in process_t. See: https://bugs.torproject.org/28179 --- src/test/test_process.c | 52 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/src/test/test_process.c b/src/test/test_process.c index 4f86e786cb..2fcb3f6686 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -9,6 +9,7 @@ #include "orconfig.h" #include "core/or/or.h" #include "test/test.h" +#include "lib/process/env.h" #define PROCESS_PRIVATE #include "lib/process/process.h" @@ -172,6 +173,56 @@ test_default_values(void *arg) process_free_all(); } +static void +test_environment(void *arg) +{ + (void)arg; + process_init(); + + process_t *process = process_new(""); + process_environment_t *env = NULL; + + process_set_environment(process, "E", "F"); + process_set_environment(process, "C", "D"); + process_set_environment(process, "A", "B"); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "A=B\0C=D\0E=F\0", 12); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "A=B"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "C=D"); + tt_str_op(env->unixoid_environment_block[2], OP_EQ, + "E=F"); + tt_ptr_op(env->unixoid_environment_block[3], OP_EQ, + NULL); + process_environment_free(env); + + /* Reset our environment. */ + smartlist_t *new_env = smartlist_new(); + smartlist_add(new_env, (char *)"FOO=bar"); + smartlist_add(new_env, (char *)"HELLO=world"); + + process_reset_environment(process, new_env); + smartlist_free(new_env); + + env = process_get_environment(process); + tt_mem_op(env->windows_environment_block, OP_EQ, + "FOO=bar\0HELLO=world\0", 20); + tt_str_op(env->unixoid_environment_block[0], OP_EQ, + "FOO=bar"); + tt_str_op(env->unixoid_environment_block[1], OP_EQ, + "HELLO=world"); + tt_ptr_op(env->unixoid_environment_block[2], OP_EQ, + NULL); + + done: + process_environment_free(env); + process_free(process); + process_free_all(); +} + static void test_stringified_types(void *arg) { @@ -589,6 +640,7 @@ test_win32(void *arg) struct testcase_t process_tests[] = { { "default_values", test_default_values, TT_FORK, NULL, NULL }, + { "environment", test_environment, TT_FORK, NULL, NULL }, { "stringified_types", test_stringified_types, TT_FORK, NULL, NULL }, { "line_protocol_simple", test_line_protocol_simple, TT_FORK, NULL, NULL }, { "line_protocol_multi", test_line_protocol_multi, TT_FORK, NULL, NULL }, From f7d13425fc738d7d56d5582b9be8510ac236c076 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 05:34:30 +0100 Subject: [PATCH 21/34] Delete old process_handle_t code. This patch removes the old process_handle_t code. Everything should by now be using the process_t interface. See: https://bugs.torproject.org/28179 --- .gitignore | 2 - src/feature/client/transports.c | 16 - src/lib/process/subprocess.c | 1078 ------------------------------- src/lib/process/subprocess.h | 118 +--- src/test/Makefile.nmake | 7 +- src/test/include.am | 2 - src/test/test-child.c | 61 -- src/test/test.h | 1 - src/test/test_logging.c | 21 +- src/test/test_slow.c | 1 - src/test/test_util.c | 262 -------- src/test/test_util_slow.c | 396 ------------ 12 files changed, 17 insertions(+), 1948 deletions(-) delete mode 100644 src/test/test-child.c delete mode 100644 src/test/test_util_slow.c diff --git a/.gitignore b/.gitignore index df8db1113a..60635263ad 100644 --- a/.gitignore +++ b/.gitignore @@ -239,7 +239,6 @@ uptime-*.json /src/test/test /src/test/test-slow /src/test/test-bt-cl -/src/test/test-child /src/test/test-process /src/test/test-memwipe /src/test/test-ntor-cl @@ -250,7 +249,6 @@ uptime-*.json /src/test/test.exe /src/test/test-slow.exe /src/test/test-bt-cl.exe -/src/test/test-child.exe /src/test/test-process.exe /src/test/test-ntor-cl.exe /src/test/test-hs-ntor-cl.exe diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index dc76e9c245..fd4bec7807 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -913,22 +913,6 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) } else if (!strcmpstart(line, PROTO_LOG)) { parse_log_line(line); return; - } else if (!strcmpstart(line, SPAWN_ERROR_MESSAGE)) { - /* managed proxy launch failed: parse error message to learn why. */ - int retval, child_state, saved_errno; - retval = tor_sscanf(line, SPAWN_ERROR_MESSAGE "%x/%x", - &child_state, &saved_errno); - if (retval == 2) { - log_warn(LD_GENERAL, - "Could not launch managed proxy executable at '%s' ('%s').", - mp->argv[0], strerror(saved_errno)); - } else { /* failed to parse error message */ - log_warn(LD_GENERAL,"Could not launch managed proxy executable at '%s'.", - mp->argv[0]); - } - - mp->conf_state = PT_PROTO_FAILED_LAUNCH; - return; } log_notice(LD_GENERAL, "Unknown line received by managed proxy (%s).", line); diff --git a/src/lib/process/subprocess.c b/src/lib/process/subprocess.c index 70851c15e0..c163790155 100644 --- a/src/lib/process/subprocess.c +++ b/src/lib/process/subprocess.c @@ -142,236 +142,6 @@ tor_join_win_cmdline(const char *argv[]) return joined_argv; } -#ifndef _WIN32 -/** Format child_state and saved_errno as a hex string placed in - * hex_errno. Called between fork and _exit, so must be signal-handler - * safe. - * - * hex_errno must have at least HEX_ERRNO_SIZE+1 bytes available. - * - * The format of hex_errno is: "CHILD_STATE/ERRNO\n", left-padded - * with spaces. CHILD_STATE indicates where - * in the process of starting the child process did the failure occur (see - * CHILD_STATE_* macros for definition), and SAVED_ERRNO is the value of - * errno when the failure occurred. - * - * On success return the number of characters added to hex_errno, not counting - * the terminating NUL; return -1 on error. - */ -STATIC int -format_helper_exit_status(unsigned char child_state, int saved_errno, - char *hex_errno) -{ - unsigned int unsigned_errno; - int written, left; - char *cur; - size_t i; - int res = -1; - - /* Fill hex_errno with spaces, and a trailing newline (memset may - not be signal handler safe, so we can't use it) */ - for (i = 0; i < (HEX_ERRNO_SIZE - 1); i++) - hex_errno[i] = ' '; - hex_errno[HEX_ERRNO_SIZE - 1] = '\n'; - - /* Convert errno to be unsigned for hex conversion */ - if (saved_errno < 0) { - // Avoid overflow on the cast to unsigned int when result is INT_MIN - // by adding 1 to the signed int negative value, - // then, after it has been negated and cast to unsigned, - // adding the original 1 back (the double-addition is intentional). - // Otherwise, the cast to signed could cause a temporary int - // to equal INT_MAX + 1, which is undefined. - unsigned_errno = ((unsigned int) -(saved_errno + 1)) + 1; - } else { - unsigned_errno = (unsigned int) saved_errno; - } - - /* - * Count how many chars of space we have left, and keep a pointer into the - * current point in the buffer. - */ - left = HEX_ERRNO_SIZE+1; - cur = hex_errno; - - /* Emit child_state */ - written = format_hex_number_sigsafe(child_state, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - if (left <= 0) - goto err; - - /* Now the '/' */ - *cur = '/'; - - /* Adjust left and cur */ - ++cur; - --left; - if (left <= 0) - goto err; - - /* Need minus? */ - if (saved_errno < 0) { - *cur = '-'; - ++cur; - --left; - if (left <= 0) - goto err; - } - - /* Emit unsigned_errno */ - written = format_hex_number_sigsafe(unsigned_errno, cur, left); - - if (written <= 0) - goto err; - - /* Adjust left and cur */ - left -= written; - cur += written; - - /* Check that we have enough space left for a newline and a NUL */ - if (left <= 1) - goto err; - - /* Emit the newline and NUL */ - *cur++ = '\n'; - *cur++ = '\0'; - - res = (int)(cur - hex_errno - 1); - - goto done; - - err: - /* - * In error exit, just write a '\0' in the first char so whatever called - * this at least won't fall off the end. - */ - *hex_errno = '\0'; - - done: - return res; -} -#endif /* !defined(_WIN32) */ - -/* Maximum number of file descriptors, if we cannot get it via sysconf() */ -#define DEFAULT_MAX_FD 256 - -/** Terminate the process of process_handle, if that process has not - * already exited. - * - * Return 0 if we succeeded in terminating the process (or if the process - * already exited), and -1 if we tried to kill the process but failed. - * - * Based on code originally borrowed from Python's os.kill. */ -int -tor_terminate_process(process_handle_t *process_handle) -{ -#ifdef _WIN32 - if (tor_get_exit_code(process_handle, 0, NULL) == PROCESS_EXIT_RUNNING) { - HANDLE handle = process_handle->pid.hProcess; - - if (!TerminateProcess(handle, 0)) - return -1; - else - return 0; - } -#else /* !(defined(_WIN32)) */ - if (process_handle->waitpid_cb) { - /* We haven't got a waitpid yet, so we can just kill off the process. */ - return kill(process_handle->pid, SIGTERM); - } -#endif /* defined(_WIN32) */ - - return 0; /* We didn't need to kill the process, so report success */ -} - -/** Return the Process ID of process_handle. */ -int -tor_process_get_pid(process_handle_t *process_handle) -{ -#ifdef _WIN32 - return (int) process_handle->pid.dwProcessId; -#else - return (int) process_handle->pid; -#endif -} - -#ifdef _WIN32 -HANDLE -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#else /* !(defined(_WIN32)) */ -/* DOCDOC tor_process_get_stdout_pipe */ -int -tor_process_get_stdout_pipe(process_handle_t *process_handle) -{ - return process_handle->stdout_pipe; -} -#endif /* defined(_WIN32) */ - -/* DOCDOC process_handle_new */ -static process_handle_t * -process_handle_new(void) -{ - process_handle_t *out = tor_malloc_zero(sizeof(process_handle_t)); - -#ifdef _WIN32 - out->stdin_pipe = INVALID_HANDLE_VALUE; - out->stdout_pipe = INVALID_HANDLE_VALUE; - out->stderr_pipe = INVALID_HANDLE_VALUE; -#else - out->stdin_pipe = -1; - out->stdout_pipe = -1; - out->stderr_pipe = -1; -#endif /* defined(_WIN32) */ - - return out; -} - -#ifndef _WIN32 -/** Invoked when a process that we've launched via tor_spawn_background() has - * been found to have terminated. - */ -static void -process_handle_waitpid_cb(int status, void *arg) -{ - process_handle_t *process_handle = arg; - - process_handle->waitpid_exit_status = status; - clear_waitpid_callback(process_handle->waitpid_cb); - if (process_handle->status == PROCESS_STATUS_RUNNING) - process_handle->status = PROCESS_STATUS_NOTRUNNING; - process_handle->waitpid_cb = 0; -} -#endif /* !defined(_WIN32) */ - -/** - * @name child-process states - * - * Each of these values represents a possible state that a child process can - * be in. They're used to determine what to say when telling the parent how - * far along we were before failure. - * - * @{ - */ -#define CHILD_STATE_INIT 0 -#define CHILD_STATE_PIPE 1 -#define CHILD_STATE_MAXFD 2 -#define CHILD_STATE_FORK 3 -#define CHILD_STATE_DUPOUT 4 -#define CHILD_STATE_DUPERR 5 -#define CHILD_STATE_DUPIN 6 -#define CHILD_STATE_CLOSEFD 7 -#define CHILD_STATE_EXEC 8 -#define CHILD_STATE_FAILEXEC 9 -/** @} */ /** * Boolean. If true, then Tor may call execve or CreateProcess via * tor_spawn_background. @@ -386,851 +156,3 @@ tor_disable_spawning_background_processes(void) { may_spawn_background_process = 0; } -/** Start a program in the background. If filename contains a '/', then - * it will be treated as an absolute or relative path. Otherwise, on - * non-Windows systems, the system path will be searched for filename. - * On Windows, only the current directory will be searched. Here, to search the - * system path (as well as the application directory, current working - * directory, and system directories), set filename to NULL. - * - * The strings in argv will be passed as the command line arguments of - * the child program (following convention, argv[0] should normally be the - * filename of the executable, and this must be the case if filename is - * NULL). The last element of argv must be NULL. A handle to the child process - * will be returned in process_handle (which must be non-NULL). Read - * process_handle.status to find out if the process was successfully launched. - * For convenience, process_handle.status is returned by this function. - * - * Some parts of this code are based on the POSIX subprocess module from - * Python, and example code from - * http://msdn.microsoft.com/en-us/library/ms682499%28v=vs.85%29.aspx. - */ -int -tor_spawn_background(const char *const filename, const char **argv, - process_environment_t *env, - process_handle_t **process_handle_out) -{ - if (BUG(may_spawn_background_process == 0)) { - /* We should never reach this point if we're forbidden to spawn - * processes. Instead we should have caught the attempt earlier. */ - return PROCESS_STATUS_ERROR; - } - -#ifdef _WIN32 - HANDLE stdout_pipe_read = NULL; - HANDLE stdout_pipe_write = NULL; - HANDLE stderr_pipe_read = NULL; - HANDLE stderr_pipe_write = NULL; - HANDLE stdin_pipe_read = NULL; - HANDLE stdin_pipe_write = NULL; - process_handle_t *process_handle; - int status; - - STARTUPINFOA siStartInfo; - BOOL retval = FALSE; - - SECURITY_ATTRIBUTES saAttr; - char *joined_argv; - - saAttr.nLength = sizeof(SECURITY_ATTRIBUTES); - saAttr.bInheritHandle = TRUE; - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - saAttr.lpSecurityDescriptor = NULL; - - /* Assume failure to start process */ - status = PROCESS_STATUS_ERROR; - - /* Set up pipe for stdout */ - if (!CreatePipe(&stdout_pipe_read, &stdout_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdout communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdout_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdout communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stderr */ - if (!CreatePipe(&stderr_pipe_read, &stderr_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stderr communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stderr_pipe_read, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stderr communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Set up pipe for stdin */ - if (!CreatePipe(&stdin_pipe_read, &stdin_pipe_write, &saAttr, 0)) { - log_warn(LD_GENERAL, - "Failed to create pipe for stdin communication with child process: %s", - format_win32_error(GetLastError())); - return status; - } - if (!SetHandleInformation(stdin_pipe_write, HANDLE_FLAG_INHERIT, 0)) { - log_warn(LD_GENERAL, - "Failed to configure pipe for stdin communication with child " - "process: %s", format_win32_error(GetLastError())); - return status; - } - - /* Create the child process */ - - /* Windows expects argv to be a whitespace delimited string, so join argv up - */ - joined_argv = tor_join_win_cmdline(argv); - - process_handle = process_handle_new(); - process_handle->status = status; - - ZeroMemory(&(process_handle->pid), sizeof(PROCESS_INFORMATION)); - ZeroMemory(&siStartInfo, sizeof(STARTUPINFO)); - siStartInfo.cb = sizeof(STARTUPINFO); - siStartInfo.hStdError = stderr_pipe_write; - siStartInfo.hStdOutput = stdout_pipe_write; - siStartInfo.hStdInput = stdin_pipe_read; - siStartInfo.dwFlags |= STARTF_USESTDHANDLES; - - /* Create the child process */ - - retval = CreateProcessA(filename, // module name - joined_argv, // command line - /* TODO: should we set explicit security attributes? (#2046, comment 5) */ - NULL, // process security attributes - NULL, // primary thread security attributes - TRUE, // handles are inherited - /*(TODO: set CREATE_NEW CONSOLE/PROCESS_GROUP to make GetExitCodeProcess() - * work?) */ - CREATE_NO_WINDOW, // creation flags - (env==NULL) ? NULL : env->windows_environment_block, - NULL, // use parent's current directory - &siStartInfo, // STARTUPINFO pointer - &(process_handle->pid)); // receives PROCESS_INFORMATION - - tor_free(joined_argv); - - if (!retval) { - log_warn(LD_GENERAL, - "Failed to create child process %s: %s", filename?filename:argv[0], - format_win32_error(GetLastError())); - tor_free(process_handle); - } else { - /* TODO: Close hProcess and hThread in process_handle->pid? */ - process_handle->stdout_pipe = stdout_pipe_read; - process_handle->stderr_pipe = stderr_pipe_read; - process_handle->stdin_pipe = stdin_pipe_write; - status = process_handle->status = PROCESS_STATUS_RUNNING; - } - - /* TODO: Close pipes on exit */ - *process_handle_out = process_handle; - return status; -#else /* !(defined(_WIN32)) */ - pid_t pid; - int stdout_pipe[2]; - int stderr_pipe[2]; - int stdin_pipe[2]; - int fd, retval; - process_handle_t *process_handle; - int status; - - const char *error_message = SPAWN_ERROR_MESSAGE; - size_t error_message_length; - - /* Represents where in the process of spawning the program is; - this is used for printing out the error message */ - unsigned char child_state = CHILD_STATE_INIT; - - char hex_errno[HEX_ERRNO_SIZE + 2]; /* + 1 should be sufficient actually */ - - static int max_fd = -1; - - status = PROCESS_STATUS_ERROR; - - /* We do the strlen here because strlen() is not signal handler safe, - and we are not allowed to use unsafe functions between fork and exec */ - error_message_length = strlen(error_message); - - // child_state = CHILD_STATE_PIPE; - - /* Set up pipe for redirecting stdout, stderr, and stdin of child */ - retval = pipe(stdout_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdout communication with child process: %s", - strerror(errno)); - return status; - } - - retval = pipe(stderr_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stderr communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - - return status; - } - - retval = pipe(stdin_pipe); - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to set up pipe for stdin communication with child process: %s", - strerror(errno)); - - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - - return status; - } - - // child_state = CHILD_STATE_MAXFD; - -#ifdef _SC_OPEN_MAX - if (-1 == max_fd) { - max_fd = (int) sysconf(_SC_OPEN_MAX); - if (max_fd == -1) { - max_fd = DEFAULT_MAX_FD; - log_warn(LD_GENERAL, - "Cannot find maximum file descriptor, assuming %d", max_fd); - } - } -#else /* !(defined(_SC_OPEN_MAX)) */ - max_fd = DEFAULT_MAX_FD; -#endif /* defined(_SC_OPEN_MAX) */ - - // child_state = CHILD_STATE_FORK; - - pid = fork(); - if (0 == pid) { - /* In child */ - -#if defined(HAVE_SYS_PRCTL_H) && defined(__linux__) - /* Attempt to have the kernel issue a SIGTERM if the parent - * goes away. Certain attributes of the binary being execve()ed - * will clear this during the execve() call, but it's better - * than nothing. - */ - prctl(PR_SET_PDEATHSIG, SIGTERM); -#endif /* defined(HAVE_SYS_PRCTL_H) && defined(__linux__) */ - - child_state = CHILD_STATE_DUPOUT; - - /* Link child stdout to the write end of the pipe */ - retval = dup2(stdout_pipe[1], STDOUT_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPERR; - - /* Link child stderr to the write end of the pipe */ - retval = dup2(stderr_pipe[1], STDERR_FILENO); - if (-1 == retval) - goto error; - - child_state = CHILD_STATE_DUPIN; - - /* Link child stdin to the read end of the pipe */ - retval = dup2(stdin_pipe[0], STDIN_FILENO); - if (-1 == retval) - goto error; - - // child_state = CHILD_STATE_CLOSEFD; - - close(stderr_pipe[0]); - close(stderr_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - - /* Close all other fds, including the read end of the pipe */ - /* XXX: We should now be doing enough FD_CLOEXEC setting to make - * this needless. */ - for (fd = STDERR_FILENO + 1; fd < max_fd; fd++) { - close(fd); - } - - // child_state = CHILD_STATE_EXEC; - - /* Call the requested program. We need the cast because - execvp doesn't define argv as const, even though it - does not modify the arguments */ - if (env) - execve(filename, (char *const *) argv, env->unixoid_environment_block); - else { - static char *new_env[] = { NULL }; - execve(filename, (char *const *) argv, new_env); - } - - /* If we got here, the exec or open(/dev/null) failed */ - - child_state = CHILD_STATE_FAILEXEC; - - error: - { - /* XXX: are we leaking fds from the pipe? */ - int n, err=0; - ssize_t nbytes; - - n = format_helper_exit_status(child_state, errno, hex_errno); - - if (n >= 0) { - /* Write the error message. GCC requires that we check the return - value, but there is nothing we can do if it fails */ - /* TODO: Don't use STDOUT, use a pipe set up just for this purpose */ - nbytes = write(STDOUT_FILENO, error_message, error_message_length); - err = (nbytes < 0); - nbytes = write(STDOUT_FILENO, hex_errno, n); - err += (nbytes < 0); - } - - _exit(err?254:255); // exit ok: in child. - } - - /* Never reached, but avoids compiler warning */ - return status; // LCOV_EXCL_LINE - } - - /* In parent */ - - if (-1 == pid) { - log_warn(LD_GENERAL, "Failed to fork child process: %s", strerror(errno)); - close(stdin_pipe[0]); - close(stdin_pipe[1]); - close(stdout_pipe[0]); - close(stdout_pipe[1]); - close(stderr_pipe[0]); - close(stderr_pipe[1]); - return status; - } - - process_handle = process_handle_new(); - process_handle->status = status; - process_handle->pid = pid; - - /* TODO: If the child process forked but failed to exec, waitpid it */ - - /* Return read end of the pipes to caller, and close write end */ - process_handle->stdout_pipe = stdout_pipe[0]; - retval = close(stdout_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stdout pipe in parent process: %s", - strerror(errno)); - } - - process_handle->waitpid_cb = set_waitpid_callback(pid, - process_handle_waitpid_cb, - process_handle); - - process_handle->stderr_pipe = stderr_pipe[0]; - retval = close(stderr_pipe[1]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close write end of stderr pipe in parent process: %s", - strerror(errno)); - } - - /* Return write end of the stdin pipe to caller, and close the read end */ - process_handle->stdin_pipe = stdin_pipe[1]; - retval = close(stdin_pipe[0]); - - if (-1 == retval) { - log_warn(LD_GENERAL, - "Failed to close read end of stdin pipe in parent process: %s", - strerror(errno)); - } - - status = process_handle->status = PROCESS_STATUS_RUNNING; - /* Set stdin/stdout/stderr pipes to be non-blocking */ - if (fcntl(process_handle->stdout_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stderr_pipe, F_SETFL, O_NONBLOCK) < 0 || - fcntl(process_handle->stdin_pipe, F_SETFL, O_NONBLOCK) < 0) { - log_warn(LD_GENERAL, "Failed to set stderror/stdout/stdin pipes " - "nonblocking in parent process: %s", strerror(errno)); - } - - *process_handle_out = process_handle; - return status; -#endif /* defined(_WIN32) */ -} - -/** Destroy all resources allocated by the process handle in - * process_handle. - * If also_terminate_process is true, also terminate the - * process of the process handle. */ -MOCK_IMPL(void, -tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)) -{ - if (!process_handle) - return; - - if (also_terminate_process) { - if (tor_terminate_process(process_handle) < 0) { - const char *errstr = -#ifdef _WIN32 - format_win32_error(GetLastError()); -#else - strerror(errno); -#endif - log_notice(LD_GENERAL, "Failed to terminate process with " - "PID '%d' ('%s').", tor_process_get_pid(process_handle), - errstr); - } else { - log_info(LD_GENERAL, "Terminated process with PID '%d'.", - tor_process_get_pid(process_handle)); - } - } - - process_handle->status = PROCESS_STATUS_NOTRUNNING; - -#ifdef _WIN32 - if (process_handle->stdout_pipe) - CloseHandle(process_handle->stdout_pipe); - - if (process_handle->stderr_pipe) - CloseHandle(process_handle->stderr_pipe); - - if (process_handle->stdin_pipe) - CloseHandle(process_handle->stdin_pipe); -#else /* !(defined(_WIN32)) */ - close(process_handle->stdout_pipe); - close(process_handle->stderr_pipe); - close(process_handle->stdin_pipe); - - clear_waitpid_callback(process_handle->waitpid_cb); -#endif /* defined(_WIN32) */ - - memset(process_handle, 0x0f, sizeof(process_handle_t)); - tor_free(process_handle); -} - -/** Get the exit code of a process specified by process_handle and store - * it in exit_code, if set to a non-NULL value. If block is set - * to true, the call will block until the process has exited. Otherwise if - * the process is still running, the function will return - * PROCESS_EXIT_RUNNING, and exit_code will be left unchanged. Returns - * PROCESS_EXIT_EXITED if the process did exit. If there is a failure, - * PROCESS_EXIT_ERROR will be returned and the contents of exit_code (if - * non-NULL) will be undefined. N.B. Under *nix operating systems, this will - * probably not work in Tor, because waitpid() is called in main.c to reap any - * terminated child processes.*/ -int -tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code) -{ -#ifdef _WIN32 - DWORD retval; - BOOL success; - - if (block) { - /* Wait for the process to exit */ - retval = WaitForSingleObject(process_handle->pid.hProcess, INFINITE); - if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } else { - retval = WaitForSingleObject(process_handle->pid.hProcess, 0); - if (WAIT_TIMEOUT == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != WAIT_OBJECT_0) { - log_warn(LD_GENERAL, "WaitForSingleObject() failed (%d): %s", - (int)retval, format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } - - if (exit_code != NULL) { - success = GetExitCodeProcess(process_handle->pid.hProcess, - (PDWORD)exit_code); - if (!success) { - log_warn(LD_GENERAL, "GetExitCodeProcess() failed: %s", - format_win32_error(GetLastError())); - return PROCESS_EXIT_ERROR; - } - } -#else /* !(defined(_WIN32)) */ - int stat_loc; - int retval; - - if (process_handle->waitpid_cb) { - /* We haven't processed a SIGCHLD yet. */ - retval = waitpid(process_handle->pid, &stat_loc, block?0:WNOHANG); - if (retval == process_handle->pid) { - clear_waitpid_callback(process_handle->waitpid_cb); - process_handle->waitpid_cb = NULL; - process_handle->waitpid_exit_status = stat_loc; - } - } else { - /* We already got a SIGCHLD for this process, and handled it. */ - retval = process_handle->pid; - stat_loc = process_handle->waitpid_exit_status; - } - - if (!block && 0 == retval) { - /* Process has not exited */ - return PROCESS_EXIT_RUNNING; - } else if (retval != process_handle->pid) { - log_warn(LD_GENERAL, "waitpid() failed for PID %d: %s", - (int)process_handle->pid, strerror(errno)); - return PROCESS_EXIT_ERROR; - } - - if (!WIFEXITED(stat_loc)) { - log_warn(LD_GENERAL, "Process %d did not exit normally", - (int)process_handle->pid); - return PROCESS_EXIT_ERROR; - } - - if (exit_code != NULL) - *exit_code = WEXITSTATUS(stat_loc); -#endif /* defined(_WIN32) */ - - return PROCESS_EXIT_EXITED; -} - -#ifdef _WIN32 -/** Read from a handle h into buf, up to count bytes. If - * hProcess is NULL, the function will return immediately if there is - * nothing more to read. Otherwise hProcess should be set to the handle - * to the process owning the h. In this case, the function will exit - * only once the process has exited, or count bytes are read. Returns - * the number of bytes read, or -1 on error. */ -ssize_t -tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process) -{ - size_t numread = 0; - BOOL retval; - DWORD byte_count; - BOOL process_exited = FALSE; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - /* Check if there is anything to read */ - retval = PeekNamedPipe(h, NULL, 0, NULL, &byte_count, NULL); - if (!retval) { - log_warn(LD_GENERAL, - "Failed to peek from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* Nothing available: process exited or it is busy */ - - /* Exit if we don't know whether the process is running */ - if (NULL == process) - break; - - /* The process exited and there's nothing left to read from it */ - if (process_exited) - break; - - /* If process is not running, check for output one more time in case - it wrote something after the peek was performed. Otherwise keep on - waiting for output */ - tor_assert(process != NULL); - byte_count = WaitForSingleObject(process->pid.hProcess, 0); - if (WAIT_TIMEOUT != byte_count) - process_exited = TRUE; - - continue; - } - - /* There is data to read; read it */ - retval = ReadFile(h, buf+numread, count-numread, &byte_count, NULL); - tor_assert(byte_count + numread <= count); - if (!retval) { - log_warn(LD_GENERAL, "Failed to read from handle: %s", - format_win32_error(GetLastError())); - return -1; - } else if (0 == byte_count) { - /* End of file */ - break; - } - numread += byte_count; - } - return (ssize_t)numread; -} -#else /* !(defined(_WIN32)) */ -/** Read from a handle fd into buf, up to count bytes. If - * process is NULL, the function will return immediately if there is - * nothing more to read. Otherwise data will be read until end of file, or - * count bytes are read. Returns the number of bytes read, or -1 on - * error. Sets eof to true if eof is not NULL and the end of the - * file has been reached. */ -ssize_t -tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof) -{ - size_t numread = 0; - ssize_t result; - - if (eof) - *eof = 0; - - if (count > SIZE_T_CEILING || count > SSIZE_MAX) - return -1; - - while (numread < count) { - result = read(fd, buf+numread, count-numread); - - if (result == 0) { - log_debug(LD_GENERAL, "read() reached end of file"); - if (eof) - *eof = 1; - break; - } else if (result < 0 && errno == EAGAIN) { - if (process) - continue; - else - break; - } else if (result < 0) { - log_warn(LD_GENERAL, "read() failed: %s", strerror(errno)); - return -1; - } - - numread += result; - } - - log_debug(LD_GENERAL, "read() read %d bytes from handle", (int)numread); - return (ssize_t)numread; -} -#endif /* defined(_WIN32) */ - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stdout(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stdout_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Read from stdout of a process until the process exits. */ -ssize_t -tor_read_all_from_process_stderr(const process_handle_t *process_handle, - char *buf, size_t count) -{ -#ifdef _WIN32 - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle); -#else - return tor_read_all_handle(process_handle->stderr_pipe, buf, count, - process_handle, NULL); -#endif /* defined(_WIN32) */ -} - -/** Return a string corresponding to stream_status. */ -const char * -stream_status_to_string(enum stream_status stream_status) -{ - switch (stream_status) { - case IO_STREAM_OKAY: - return "okay"; - case IO_STREAM_EAGAIN: - return "temporarily unavailable"; - case IO_STREAM_TERM: - return "terminated"; - case IO_STREAM_CLOSED: - return "closed"; - default: - tor_fragile_assert(); - return "unknown"; - } -} - -/** Split buf into lines, and add to smartlist. The buffer buf will be - * modified. The resulting smartlist will consist of pointers to buf, so there - * is no need to free the contents of sl. buf must be a NUL-terminated - * string. len should be set to the length of the buffer excluding the - * NUL. Non-printable characters (including NUL) will be replaced with "." */ -int -tor_split_lines(smartlist_t *sl, char *buf, int len) -{ - /* Index in buf of the start of the current line */ - int start = 0; - /* Index in buf of the current character being processed */ - int cur = 0; - /* Are we currently in a line */ - char in_line = 0; - - /* Loop over string */ - while (cur < len) { - /* Loop until end of line or end of string */ - for (; cur < len; cur++) { - if (in_line) { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* End of line */ - buf[cur] = '\0'; - /* Point cur to the next line */ - cur++; - /* Line starts at start and ends with a nul */ - break; - } else { - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } else { - if ('\r' == buf[cur] || '\n' == buf[cur]) { - /* Skip leading vertical space */ - ; - } else { - in_line = 1; - start = cur; - if (!TOR_ISPRINT(buf[cur])) - buf[cur] = '.'; - } - } - } - /* We are at the end of the line or end of string. If in_line is true there - * is a line which starts at buf+start and ends at a NUL. cur points to - * the character after the NUL. */ - if (in_line) - smartlist_add(sl, (void *)(buf+start)); - in_line = 0; - } - return smartlist_len(sl); -} - -#ifdef _WIN32 - -/** Return a smartlist containing lines outputted from - * handle. Return NULL on error, and set - * stream_status_out appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (HANDLE *handle, - enum stream_status *stream_status_out)) -{ - int pos; - char stdout_buf[600] = {0}; - smartlist_t *lines = NULL; - - tor_assert(stream_status_out); - - *stream_status_out = IO_STREAM_TERM; - - pos = tor_read_all_handle(handle, stdout_buf, sizeof(stdout_buf) - 1, NULL); - if (pos < 0) { - *stream_status_out = IO_STREAM_TERM; - return NULL; - } - if (pos == 0) { - *stream_status_out = IO_STREAM_EAGAIN; - return NULL; - } - - /* End with a null even if there isn't a \r\n at the end */ - /* TODO: What if this is a partial line? */ - stdout_buf[pos] = '\0'; - - /* Split up the buffer */ - lines = smartlist_new(); - tor_split_lines(lines, stdout_buf, pos); - - /* Currently 'lines' is populated with strings residing on the - stack. Replace them with their exact copies on the heap: */ - SMARTLIST_FOREACH(lines, char *, line, - SMARTLIST_REPLACE_CURRENT(lines, line, tor_strdup(line))); - - *stream_status_out = IO_STREAM_OKAY; - - return lines; -} - -#else /* !(defined(_WIN32)) */ - -/** Return a smartlist containing lines outputted from - * fd. Return NULL on error, and set - * stream_status_out appropriately. */ -MOCK_IMPL(smartlist_t *, -tor_get_lines_from_handle, (int fd, enum stream_status *stream_status_out)) -{ - enum stream_status stream_status; - char stdout_buf[400]; - smartlist_t *lines = NULL; - - while (1) { - memset(stdout_buf, 0, sizeof(stdout_buf)); - - stream_status = get_string_from_pipe(fd, - stdout_buf, sizeof(stdout_buf) - 1); - if (stream_status != IO_STREAM_OKAY) - goto done; - - if (!lines) lines = smartlist_new(); - smartlist_split_string(lines, stdout_buf, "\n", 0, 0); - } - - done: - *stream_status_out = stream_status; - return lines; -} - -#endif /* defined(_WIN32) */ - -/** Reads from fd and stores input in buf_out making - * sure it's below count bytes. - * If the string has a trailing newline, we strip it off. - * - * This function is specifically created to handle input from managed - * proxies, according to the pluggable transports spec. Make sure it - * fits your needs before using it. - * - * Returns: - * IO_STREAM_CLOSED: If the stream is closed. - * IO_STREAM_EAGAIN: If there is nothing to read and we should check back - * later. - * IO_STREAM_TERM: If something is wrong with the stream. - * IO_STREAM_OKAY: If everything went okay and we got a string - * in buf_out. */ -enum stream_status -get_string_from_pipe(int fd, char *buf_out, size_t count) -{ - ssize_t ret; - - tor_assert(count <= INT_MAX); - - ret = read(fd, buf_out, count); - - if (ret == 0) - return IO_STREAM_CLOSED; - else if (ret < 0 && errno == EAGAIN) - return IO_STREAM_EAGAIN; - else if (ret < 0) - return IO_STREAM_TERM; - - if (buf_out[ret - 1] == '\n') { - /* Remove the trailing newline */ - buf_out[ret - 1] = '\0'; - } else - buf_out[ret] = '\0'; - - return IO_STREAM_OKAY; -} diff --git a/src/lib/process/subprocess.h b/src/lib/process/subprocess.h index 5b4318ef2b..eb91626343 100644 --- a/src/lib/process/subprocess.h +++ b/src/lib/process/subprocess.h @@ -11,124 +11,10 @@ #ifndef TOR_SUBPROCESS_H #define TOR_SUBPROCESS_H -#include "lib/cc/torint.h" -#include "lib/testsupport/testsupport.h" -#include -#ifdef _WIN32 -#include -#endif - -struct smartlist_t; - void tor_disable_spawning_background_processes(void); -typedef struct process_handle_t process_handle_t; -struct process_environment_t; -int tor_spawn_background(const char *const filename, const char **argv, - struct process_environment_t *env, - process_handle_t **process_handle_out); - -#define SPAWN_ERROR_MESSAGE "ERR: Failed to spawn background process - code " - -/** Status of an I/O stream. */ -enum stream_status { - IO_STREAM_OKAY, - IO_STREAM_EAGAIN, - IO_STREAM_TERM, - IO_STREAM_CLOSED -}; - -const char *stream_status_to_string(enum stream_status stream_status); - -enum stream_status get_string_from_pipe(int fd, char *buf, size_t count); - -/* Values of process_handle_t.status. */ -#define PROCESS_STATUS_NOTRUNNING 0 -#define PROCESS_STATUS_RUNNING 1 -#define PROCESS_STATUS_ERROR -1 - -#ifdef SUBPROCESS_PRIVATE -struct waitpid_callback_t; - -/** Structure to represent the state of a process with which Tor is - * communicating. The contents of this structure are private to util.c */ -struct process_handle_t { - /** One of the PROCESS_STATUS_* values */ - int status; -#ifdef _WIN32 - HANDLE stdin_pipe; - HANDLE stdout_pipe; - HANDLE stderr_pipe; - PROCESS_INFORMATION pid; -#else /* !(defined(_WIN32)) */ - int stdin_pipe; - int stdout_pipe; - int stderr_pipe; - pid_t pid; - /** If the process has not given us a SIGCHLD yet, this has the - * waitpid_callback_t that gets invoked once it has. Otherwise this - * contains NULL. */ - struct waitpid_callback_t *waitpid_cb; - /** The exit status reported by waitpid. */ - int waitpid_exit_status; -#endif /* defined(_WIN32) */ -}; -#endif /* defined(SUBPROCESS_PRIVATE) */ - -/* Return values of tor_get_exit_code() */ -#define PROCESS_EXIT_RUNNING 1 -#define PROCESS_EXIT_EXITED 0 -#define PROCESS_EXIT_ERROR -1 -int tor_get_exit_code(process_handle_t *process_handle, - int block, int *exit_code); -int tor_split_lines(struct smartlist_t *sl, char *buf, int len); -#ifdef _WIN32 -ssize_t tor_read_all_handle(HANDLE h, char *buf, size_t count, - const process_handle_t *process); -#else -ssize_t tor_read_all_handle(int fd, char *buf, size_t count, - const process_handle_t *process, - int *eof); -#endif /* defined(_WIN32) */ -ssize_t tor_read_all_from_process_stdout( - const process_handle_t *process_handle, char *buf, size_t count); -ssize_t tor_read_all_from_process_stderr( - const process_handle_t *process_handle, char *buf, size_t count); -char *tor_join_win_cmdline(const char *argv[]); - -int tor_process_get_pid(process_handle_t *process_handle); -#ifdef _WIN32 -HANDLE tor_process_get_stdout_pipe(process_handle_t *process_handle); -#else -int tor_process_get_stdout_pipe(process_handle_t *process_handle); -#endif - -#ifdef _WIN32 -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(HANDLE *handle, - enum stream_status *stream_status)); -#else -MOCK_DECL(struct smartlist_t *, tor_get_lines_from_handle,(int fd, - enum stream_status *stream_status)); -#endif /* defined(_WIN32) */ - -int tor_terminate_process(process_handle_t *process_handle); - -MOCK_DECL(void, tor_process_handle_destroy,(process_handle_t *process_handle, - int also_terminate_process)); - -#ifdef SUBPROCESS_PRIVATE -/* Prototypes for private functions only used by util.c (and unit tests) */ - #ifndef _WIN32 -STATIC int format_helper_exit_status(unsigned char child_state, - int saved_errno, char *hex_errno); - -/* Space for hex values of child state, a slash, saved_errno (with - leading minus) and newline (no null) */ -#define HEX_ERRNO_SIZE (sizeof(char) * 2 + 1 + \ - 1 + sizeof(int) * 2 + 1) -#endif /* !defined(_WIN32) */ - -#endif /* defined(SUBPROCESS_PRIVATE) */ +char *tor_join_win_cmdline(const char *argv[]); +#endif #endif diff --git a/src/test/Makefile.nmake b/src/test/Makefile.nmake index cfbe281b94..aa16a22b52 100644 --- a/src/test/Makefile.nmake +++ b/src/test/Makefile.nmake @@ -1,4 +1,4 @@ -all: test.exe test-child.exe bench.exe +all: test.exe bench.exe CFLAGS = /I ..\win32 /I ..\..\..\build-alpha\include /I ..\common /I ..\or \ /I ..\ext @@ -30,8 +30,5 @@ test.exe: $(TEST_OBJECTS) bench.exe: bench.obj $(CC) $(CFLAGS) bench.obj $(LIBS) ..\common\*.lib /Fe$@ -test-child.exe: test-child.obj - $(CC) $(CFLAGS) test-child.obj /Fe$@ - clean: - del *.obj *.lib test.exe bench.exe test-child.exe + del *.obj *.lib test.exe bench.exe diff --git a/src/test/include.am b/src/test/include.am index 9df2fd481d..e1f55e5d03 100644 --- a/src/test/include.am +++ b/src/test/include.am @@ -65,7 +65,6 @@ noinst_PROGRAMS+= \ src/test/test \ src/test/test-slow \ src/test/test-memwipe \ - src/test/test-child \ src/test/test-process \ src/test/test_workqueue \ src/test/test-switch-id \ @@ -204,7 +203,6 @@ src_test_test_slow_SOURCES += \ src/test/test_slow.c \ src/test/test_crypto_slow.c \ src/test/test_process_slow.c \ - src/test/test_util_slow.c \ src/test/testing_common.c \ src/test/testing_rsakeys.c \ src/ext/tinytest.c diff --git a/src/test/test-child.c b/src/test/test-child.c deleted file mode 100644 index 14df1a9b76..0000000000 --- a/src/test/test-child.c +++ /dev/null @@ -1,61 +0,0 @@ -/* Copyright (c) 2011-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#include -#ifdef _WIN32 -#define WINDOWS_LEAN_AND_MEAN -#include -#else -#include -#endif /* defined(_WIN32) */ -#include - -#ifdef _WIN32 -#define SLEEP(sec) Sleep((sec)*1000) -#else -#define SLEEP(sec) sleep(sec) -#endif - -/** Trivial test program which prints out its command line arguments so we can - * check if tor_spawn_background() works */ -int -main(int argc, char **argv) -{ - int i; - int delay = 1; - int fast = 0; - - if (argc > 1) { - if (!strcmp(argv[1], "--hang")) { - delay = 60; - } else if (!strcmp(argv[1], "--fast")) { - fast = 1; - delay = 0; - } - } - - fprintf(stdout, "OUT\n"); - fprintf(stderr, "ERR\n"); - for (i = 1; i < argc; i++) - fprintf(stdout, "%s\n", argv[i]); - if (!fast) - fprintf(stdout, "SLEEPING\n"); - /* We need to flush stdout so that test_util_spawn_background_partial_read() - succeed. Otherwise ReadFile() will get the entire output in one */ - // XXX: Can we make stdio flush on newline? - fflush(stdout); - if (!fast) - SLEEP(1); - fprintf(stdout, "DONE\n"); - fflush(stdout); - if (fast) - return 0; - - while (--delay) { - SLEEP(1); - } - - return 0; -} - diff --git a/src/test/test.h b/src/test/test.h index 9f72623968..ce50fa89d9 100644 --- a/src/test/test.h +++ b/src/test/test.h @@ -271,7 +271,6 @@ extern struct testcase_t x509_tests[]; extern struct testcase_t slow_crypto_tests[]; extern struct testcase_t slow_process_tests[]; -extern struct testcase_t slow_util_tests[]; extern struct testgroup_t testgroups[]; diff --git a/src/test/test_logging.c b/src/test/test_logging.c index cd685a4af7..9d3ee5d55f 100644 --- a/src/test/test_logging.c +++ b/src/test/test_logging.c @@ -117,22 +117,27 @@ test_sigsafe_err(void *arg) content = read_file_to_str(fn, 0, NULL); tt_ptr_op(content, OP_NE, NULL); - tor_split_lines(lines, content, (int)strlen(content)); + smartlist_split_string(lines, content, "\n", 0, 0); tt_int_op(smartlist_len(lines), OP_GE, 5); - if (strstr(smartlist_get(lines, 0), "opening new log file")) + if (strstr(smartlist_get(lines, 0), "opening new log file")) { + void *item = smartlist_get(lines, 0); smartlist_del_keeporder(lines, 0); + tor_free(item); + } + tt_assert(strstr(smartlist_get(lines, 0), "Say, this isn't too cool")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 1), "==============")); - tt_assert(!strcmpstart(smartlist_get(lines, 2), "Minimal.")); - /* Next line is blank. */ - tt_assert(!strcmpstart(smartlist_get(lines, 3), "==============")); - tt_str_op(smartlist_get(lines, 4), OP_EQ, + tt_str_op(smartlist_get(lines, 1), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 2), "==============")); + tt_assert(!strcmpstart(smartlist_get(lines, 3), "Minimal.")); + tt_str_op(smartlist_get(lines, 4), OP_EQ, ""); + tt_assert(!strcmpstart(smartlist_get(lines, 5), "==============")); + tt_str_op(smartlist_get(lines, 6), OP_EQ, "Testing any attempt to manually log from a signal."); done: tor_free(content); + SMARTLIST_FOREACH(lines, char *, x, tor_free(x)); smartlist_free(lines); } diff --git a/src/test/test_slow.c b/src/test/test_slow.c index 0c66eff930..97c2912af6 100644 --- a/src/test/test_slow.c +++ b/src/test/test_slow.c @@ -21,7 +21,6 @@ struct testgroup_t testgroups[] = { { "slow/crypto/", slow_crypto_tests }, { "slow/process/", slow_process_tests }, - { "slow/util/", slow_util_tests }, END_OF_GROUPS }; diff --git a/src/test/test_util.c b/src/test/test_util.c index bcface64fd..4fa67b6419 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -4301,204 +4301,6 @@ test_util_load_win_lib(void *ptr) } #endif /* defined(_WIN32) */ -#ifndef _WIN32 -static void -clear_hex_errno(char *hex_errno) -{ - memset(hex_errno, '\0', HEX_ERRNO_SIZE + 1); -} - -static void -test_util_exit_status(void *ptr) -{ - /* Leave an extra byte for a \0 so we can do string comparison */ - char hex_errno[HEX_ERRNO_SIZE + 1]; - int n; - - (void)ptr; - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0, hex_errno); - tt_str_op("0/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - -#if SIZEOF_INT == 4 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x80000000, hex_errno); - tt_str_op("FF/-80000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#elif SIZEOF_INT == 8 - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0, 0x7FFFFFFFFFFFFFFF, hex_errno); - tt_str_op("0/7FFFFFFFFFFFFFFF\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0xFF, -0x8000000000000000, hex_errno); - tt_str_op("FF/-8000000000000000\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - tt_int_op(n,OP_EQ, HEX_ERRNO_SIZE); - -#endif /* SIZEOF_INT == 4 || ... */ - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x7F, 0, hex_errno); - tt_str_op("7F/0\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - n = format_helper_exit_status(0x08, -0x242, hex_errno); - tt_str_op("8/-242\n",OP_EQ, hex_errno); - tt_int_op(n,OP_EQ, strlen(hex_errno)); - - clear_hex_errno(hex_errno); - tt_str_op("",OP_EQ, hex_errno); - - done: - ; -} -#endif /* !defined(_WIN32) */ - -#ifndef _WIN32 -static void -test_util_string_from_pipe(void *ptr) -{ - int test_pipe[2] = {-1, -1}; - int retval = 0; - enum stream_status status = IO_STREAM_TERM; - ssize_t retlen; - char buf[4] = { 0 }; - - (void)ptr; - - errno = 0; - - /* Set up a pipe to test on */ - retval = pipe(test_pipe); - tt_int_op(retval, OP_EQ, 0); - - /* Send in a string. */ - retlen = write(test_pipe[1], "ABC", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "ABC"); - errno = 0; - - /* Send in a string that contains a nul. */ - retlen = write(test_pipe[1], "AB\0", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a nul only. */ - retlen = write(test_pipe[1], "\0", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string that contains a trailing newline. */ - retlen = write(test_pipe[1], "AB\n", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Send in a string that contains a newline only. */ - retlen = write(test_pipe[1], "\n", 1); - tt_int_op(retlen, OP_EQ, 1); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, ""); - errno = 0; - - /* Send in a string and check that we nul terminate return values. */ - retlen = write(test_pipe[1], "AAA", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AAA"); - tt_mem_op(buf, OP_EQ, "AAA\0", sizeof(buf)); - errno = 0; - - retlen = write(test_pipe[1], "B", 1); - tt_int_op(retlen, OP_EQ, 1); - - memset(buf, '\xff', sizeof(buf)); - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "B"); - tt_mem_op(buf, OP_EQ, "B\0\xff\xff", sizeof(buf)); - errno = 0; - - /* Send in multiple lines. */ - retlen = write(test_pipe[1], "A\nB", 3); - tt_int_op(retlen, OP_EQ, 3); - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "A\nB"); - errno = 0; - - /* Send in a line and close */ - retlen = write(test_pipe[1], "AB", 2); - tt_int_op(retlen, OP_EQ, 2); - retval = close(test_pipe[1]); - tt_int_op(retval, OP_EQ, 0); - test_pipe[1] = -1; - - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_OKAY); - tt_str_op(buf, OP_EQ, "AB"); - errno = 0; - - /* Check for EOF */ - status = get_string_from_pipe(test_pipe[0], buf, sizeof(buf)-1); - tt_int_op(errno, OP_EQ, 0); - tt_int_op(status, OP_EQ, IO_STREAM_CLOSED); - errno = 0; - - done: - if (test_pipe[0] != -1) - close(test_pipe[0]); - if (test_pipe[1] != -1) - close(test_pipe[1]); -} - -#endif /* !defined(_WIN32) */ - /** * Test for format_hex_number_sigsafe() */ @@ -4651,67 +4453,6 @@ struct split_lines_test_t { const char *split_line[MAX_SPLIT_LINE_COUNT]; // Split lines }; -/** - * Test that we properly split a buffer into lines - */ -static void -test_util_split_lines(void *ptr) -{ - /* Test cases. orig_line of last test case must be NULL. - * The last element of split_line[i] must be NULL. */ - struct split_lines_test_t tests[] = { - {"", 0, {NULL}}, - {"foo", 3, {"foo", NULL}}, - {"\n\rfoo\n\rbar\r\n", 12, {"foo", "bar", NULL}}, - {"fo o\r\nb\tar", 10, {"fo o", "b.ar", NULL}}, - {"\x0f""f\0o\0\n\x01""b\0r\0\r", 12, {".f.o.", ".b.r.", NULL}}, - {"line 1\r\nline 2", 14, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\nline 2", 16, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\r\r\nline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\n\n\n\rline 2", 18, {"line 1", "line 2", NULL}}, - {"line 1\r\n\r\t\r\nline 3", 18, {"line 1", ".", "line 3", NULL}}, - {"\n\t\r\t\nline 3", 11, {".", ".", "line 3", NULL}}, - {NULL, 0, { NULL }} - }; - - int i, j; - char *orig_line=NULL; - smartlist_t *sl=NULL; - - (void)ptr; - - for (i=0; tests[i].orig_line; i++) { - sl = smartlist_new(); - /* Allocate space for string and trailing NULL */ - orig_line = tor_memdup(tests[i].orig_line, tests[i].orig_length + 1); - tor_split_lines(sl, orig_line, tests[i].orig_length); - - j = 0; - log_info(LD_GENERAL, "Splitting test %d of length %d", - i, tests[i].orig_length); - SMARTLIST_FOREACH_BEGIN(sl, const char *, line) { - /* Check we have not got too many lines */ - tt_int_op(MAX_SPLIT_LINE_COUNT, OP_GT, j); - /* Check that there actually should be a line here */ - tt_ptr_op(tests[i].split_line[j], OP_NE, NULL); - log_info(LD_GENERAL, "Line %d of test %d, should be <%s>", - j, i, tests[i].split_line[j]); - /* Check that the line is as expected */ - tt_str_op(line,OP_EQ, tests[i].split_line[j]); - j++; - } SMARTLIST_FOREACH_END(line); - /* Check that we didn't miss some lines */ - tt_ptr_op(NULL,OP_EQ, tests[i].split_line[j]); - tor_free(orig_line); - smartlist_free(sl); - sl = NULL; - } - - done: - tor_free(orig_line); - smartlist_free(sl); -} - static void test_util_di_ops(void *arg) { @@ -6483,12 +6224,9 @@ struct testcase_t util_tests[] = { UTIL_TEST(nowrap_math, 0), UTIL_TEST(num_cpus, 0), UTIL_TEST_WIN_ONLY(load_win_lib, 0), - UTIL_TEST_NO_WIN(exit_status, 0), - UTIL_TEST_NO_WIN(string_from_pipe, 0), UTIL_TEST(format_hex_number, 0), UTIL_TEST(format_dec_number, 0), UTIL_TEST(join_win_cmdline, 0), - UTIL_TEST(split_lines, 0), UTIL_TEST(n_bits_set, 0), UTIL_TEST(eat_whitespace, 0), UTIL_TEST(sl_new_from_text_lines, 0), diff --git a/src/test/test_util_slow.c b/src/test/test_util_slow.c deleted file mode 100644 index c7b3e3e2a4..0000000000 --- a/src/test/test_util_slow.c +++ /dev/null @@ -1,396 +0,0 @@ -/* Copyright (c) 2001-2004, Roger Dingledine. - * Copyright (c) 2004-2006, Roger Dingledine, Nick Mathewson. - * Copyright (c) 2007-2018, The Tor Project, Inc. */ -/* See LICENSE for licensing information */ - -#include "orconfig.h" -#define UTIL_PRIVATE -#define SUBPROCESS_PRIVATE -#include "lib/crypt_ops/crypto_cipher.h" -#include "lib/log/log.h" -#include "lib/process/subprocess.h" -#include "lib/process/waitpid.h" -#include "lib/string/printf.h" -#include "lib/time/compat_time.h" -#include "test/test.h" - -#include -#include - -#ifndef BUILDDIR -#define BUILDDIR "." -#endif - -#ifdef _WIN32 -#define notify_pending_waitpid_callbacks() STMT_NIL -#define TEST_CHILD "test-child.exe" -#define EOL "\r\n" -#else -#define TEST_CHILD (BUILDDIR "/src/test/test-child") -#define EOL "\n" -#endif /* defined(_WIN32) */ - -#ifdef _WIN32 -/* I've assumed Windows doesn't have the gap between fork and exec - * that causes the race condition on unix-like platforms */ -#define MATCH_PROCESS_STATUS(s1,s2) ((s1) == (s2)) - -#else /* !(defined(_WIN32)) */ -/* work around a race condition of the timing of SIGCHLD handler updates - * to the process_handle's fields, and checks of those fields - * - * TODO: Once we can signal failure to exec, change PROCESS_STATUS_RUNNING to - * PROCESS_STATUS_ERROR (and similarly with *_OR_NOTRUNNING) */ -#define PROCESS_STATUS_RUNNING_OR_NOTRUNNING (PROCESS_STATUS_RUNNING+1) -#define IS_RUNNING_OR_NOTRUNNING(s) \ - ((s) == PROCESS_STATUS_RUNNING || (s) == PROCESS_STATUS_NOTRUNNING) -/* well, this is ugly */ -#define MATCH_PROCESS_STATUS(s1,s2) \ - ( (s1) == (s2) \ - ||((s1) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s2)) \ - ||((s2) == PROCESS_STATUS_RUNNING_OR_NOTRUNNING \ - && IS_RUNNING_OR_NOTRUNNING(s1))) - -#endif /* defined(_WIN32) */ - -/** Helper function for testing tor_spawn_background */ -static void -run_util_spawn_background(const char *argv[], const char *expected_out, - const char *expected_err, int expected_exit, - int expected_status) -{ - int retval, exit_code; - ssize_t pos; - process_handle_t *process_handle=NULL; - char stdout_buf[100], stderr_buf[100]; - int status; - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - notify_pending_waitpid_callbacks(); - - /* the race condition doesn't affect status, - * because status isn't updated by the SIGCHLD handler, - * but we still need to handle PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, status)); - if (status == PROCESS_STATUS_ERROR) { - tt_ptr_op(process_handle, OP_EQ, NULL); - return; - } - - tt_ptr_op(process_handle, OP_NE, NULL); - - /* When a spawned process forks, fails, then exits very quickly, - * (this typically occurs when exec fails) - * there is a race condition between the SIGCHLD handler - * updating the process_handle's fields, and this test - * checking the process status in those fields. - * The SIGCHLD update can occur before or after the code below executes. - * This causes intermittent failures in spawn_background_fail(), - * typically when the machine is under load. - * We use PROCESS_STATUS_RUNNING_OR_NOTRUNNING to avoid this issue. */ - - /* the race condition affects the change in - * process_handle->status from RUNNING to NOTRUNNING */ - tt_assert(MATCH_PROCESS_STATUS(expected_status, process_handle->status)); - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - /* the race condition affects the change in - * process_handle->waitpid_cb to NULL, - * so we skip the check if expected_status is ambiguous, - * that is, PROCESS_STATUS_RUNNING_OR_NOTRUNNING */ - tt_assert(process_handle->waitpid_cb != NULL - || expected_status == PROCESS_STATUS_RUNNING_OR_NOTRUNNING); -#endif /* !defined(_WIN32) */ - -#ifdef _WIN32 - tt_assert(process_handle->stdout_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stderr_pipe != INVALID_HANDLE_VALUE); - tt_assert(process_handle->stdin_pipe != INVALID_HANDLE_VALUE); -#else - tt_assert(process_handle->stdout_pipe >= 0); - tt_assert(process_handle->stderr_pipe >= 0); - tt_assert(process_handle->stdin_pipe >= 0); -#endif /* defined(_WIN32) */ - - /* Check stdout */ - pos = tor_read_all_from_process_stdout(process_handle, stdout_buf, - sizeof(stdout_buf) - 1); - tt_assert(pos >= 0); - stdout_buf[pos] = '\0'; - tt_int_op(strlen(expected_out),OP_EQ, pos); - tt_str_op(expected_out,OP_EQ, stdout_buf); - - notify_pending_waitpid_callbacks(); - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - // TODO: Make test-child exit with something other than 0 - -#ifndef _WIN32 - notify_pending_waitpid_callbacks(); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - notify_pending_waitpid_callbacks(); - - done: - if (process_handle) - tor_process_handle_destroy(process_handle, 1); -} - -/** Check that we can launch a process and read the output */ -static void -test_util_spawn_background_ok(void *ptr) -{ - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out = "OUT"EOL "--test"EOL "SLEEPING"EOL "DONE" EOL; - const char *expected_err = "ERR"EOL; - - (void)ptr; - - run_util_spawn_background(argv, expected_out, expected_err, 0, - PROCESS_STATUS_RUNNING); -} - -/** Check that failing to find the executable works as expected */ -static void -test_util_spawn_background_fail(void *ptr) -{ - const char *argv[] = {BUILDDIR "/src/test/no-such-file", "--test", NULL}; - const char *expected_err = ""; - char expected_out[1024]; - char code[32]; -#ifdef _WIN32 - const int expected_status = PROCESS_STATUS_ERROR; -#else - /* TODO: Once we can signal failure to exec, set this to be - * PROCESS_STATUS_RUNNING_OR_ERROR */ - const int expected_status = PROCESS_STATUS_RUNNING_OR_NOTRUNNING; -#endif /* defined(_WIN32) */ - - memset(expected_out, 0xf0, sizeof(expected_out)); - memset(code, 0xf0, sizeof(code)); - - (void)ptr; - - tor_snprintf(code, sizeof(code), "%x/%x", - 9 /* CHILD_STATE_FAILEXEC */ , ENOENT); - tor_snprintf(expected_out, sizeof(expected_out), - "ERR: Failed to spawn background process - code %s\n", code); - - run_util_spawn_background(argv, expected_out, expected_err, 255, - expected_status); -} - -/** Test that reading from a handle returns a partial read rather than - * blocking */ -static void -test_util_spawn_background_partial_read_impl(int exit_early) -{ - const int expected_exit = 0; - const int expected_status = PROCESS_STATUS_RUNNING; - - int retval, exit_code; - ssize_t pos = -1; - process_handle_t *process_handle=NULL; - int status; - char stdout_buf[100], stderr_buf[100]; - - const char *argv[] = {TEST_CHILD, "--test", NULL}; - const char *expected_out[] = { "OUT" EOL "--test" EOL "SLEEPING" EOL, - "DONE" EOL, - NULL }; - const char *expected_err = "ERR" EOL; - -#ifndef _WIN32 - int eof = 0; -#endif - int expected_out_ctr; - - if (exit_early) { - argv[1] = "--hang"; - expected_out[0] = "OUT"EOL "--hang"EOL "SLEEPING" EOL; - } - - /* Start the program */ -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - tt_int_op(expected_status,OP_EQ, status); - tt_assert(process_handle); - tt_int_op(expected_status,OP_EQ, process_handle->status); - - /* Check stdout */ - for (expected_out_ctr = 0; expected_out[expected_out_ctr] != NULL;) { -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL); -#else - /* Check that we didn't read the end of file last time */ - tt_assert(!eof); - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, NULL, &eof); -#endif /* defined(_WIN32) */ - log_info(LD_GENERAL, "tor_read_all_handle() returned %d", (int)pos); - - /* We would have blocked, keep on trying */ - if (0 == pos) - continue; - - tt_assert(pos > 0); - stdout_buf[pos] = '\0'; - tt_str_op(expected_out[expected_out_ctr],OP_EQ, stdout_buf); - tt_int_op(strlen(expected_out[expected_out_ctr]),OP_EQ, pos); - expected_out_ctr++; - } - - if (exit_early) { - tor_process_handle_destroy(process_handle, 1); - process_handle = NULL; - goto done; - } - - /* The process should have exited without writing more */ -#ifdef _WIN32 - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle); - tt_int_op(0,OP_EQ, pos); -#else /* !(defined(_WIN32)) */ - if (!eof) { - /* We should have got all the data, but maybe not the EOF flag */ - pos = tor_read_all_handle(process_handle->stdout_pipe, stdout_buf, - sizeof(stdout_buf) - 1, - process_handle, &eof); - tt_int_op(0,OP_EQ, pos); - tt_assert(eof); - } - /* Otherwise, we got the EOF on the last read */ -#endif /* defined(_WIN32) */ - - /* Check it terminated correctly */ - retval = tor_get_exit_code(process_handle, 1, &exit_code); - tt_int_op(PROCESS_EXIT_EXITED,OP_EQ, retval); - tt_int_op(expected_exit,OP_EQ, exit_code); - - // TODO: Make test-child exit with something other than 0 - - /* Check stderr */ - pos = tor_read_all_from_process_stderr(process_handle, stderr_buf, - sizeof(stderr_buf) - 1); - tt_assert(pos >= 0); - stderr_buf[pos] = '\0'; - tt_str_op(expected_err,OP_EQ, stderr_buf); - tt_int_op(strlen(expected_err),OP_EQ, pos); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -static void -test_util_spawn_background_partial_read(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(0); -} - -static void -test_util_spawn_background_exit_early(void *arg) -{ - (void)arg; - test_util_spawn_background_partial_read_impl(1); -} - -static void -test_util_spawn_background_waitpid_notify(void *arg) -{ - int retval, exit_code; - process_handle_t *process_handle=NULL; - int status; - int ms_timer; - - const char *argv[] = {TEST_CHILD, "--fast", NULL}; - - (void) arg; - -#ifdef _WIN32 - status = tor_spawn_background(NULL, argv, NULL, &process_handle); -#else - status = tor_spawn_background(argv[0], argv, NULL, &process_handle); -#endif - - tt_int_op(status, OP_EQ, PROCESS_STATUS_RUNNING); - tt_ptr_op(process_handle, OP_NE, NULL); - - /* We're not going to look at the stdout/stderr output this time. Instead, - * we're testing whether notify_pending_waitpid_calbacks() can report the - * process exit (on unix) and/or whether tor_get_exit_code() can notice it - * (on windows) */ - -#ifndef _WIN32 - ms_timer = 30*1000; - tt_ptr_op(process_handle->waitpid_cb, OP_NE, NULL); - while (process_handle->waitpid_cb && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - notify_pending_waitpid_callbacks(); - } - tt_int_op(ms_timer, OP_GT, 0); - tt_ptr_op(process_handle->waitpid_cb, OP_EQ, NULL); -#endif /* !defined(_WIN32) */ - - ms_timer = 30*1000; - while (((retval = tor_get_exit_code(process_handle, 0, &exit_code)) - == PROCESS_EXIT_RUNNING) && ms_timer > 0) { - tor_sleep_msec(100); - ms_timer -= 100; - } - tt_int_op(ms_timer, OP_GT, 0); - - tt_int_op(retval, OP_EQ, PROCESS_EXIT_EXITED); - - done: - tor_process_handle_destroy(process_handle, 1); -} - -#undef TEST_CHILD -#undef EOL - -#undef MATCH_PROCESS_STATUS - -#ifndef _WIN32 -#undef PROCESS_STATUS_RUNNING_OR_NOTRUNNING -#undef IS_RUNNING_OR_NOTRUNNING -#endif - -#define UTIL_TEST(name, flags) \ - { #name, test_util_ ## name, flags, NULL, NULL } - -struct testcase_t slow_util_tests[] = { - UTIL_TEST(spawn_background_ok, 0), - UTIL_TEST(spawn_background_fail, 0), - UTIL_TEST(spawn_background_partial_read, 0), - UTIL_TEST(spawn_background_exit_early, 0), - UTIL_TEST(spawn_background_waitpid_notify, 0), - END_OF_TESTCASES -}; From ccc1963890bc7448383cd4c45370139697973d64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 06:14:47 +0100 Subject: [PATCH 22/34] Move remaining code from subprocess.{h,c} to more appropriate places. This patch moves the remaining code from subprocess.{h,c} to more appropriate places in the process.c and process_win32.c module. We also delete the now empty subprocess module files. See: https://bugs.torproject.org/28179 --- src/app/config/config.c | 2 +- src/feature/client/transports.c | 1 - src/lib/process/include.am | 2 - src/lib/process/process.c | 19 ++++ src/lib/process/process.h | 2 + src/lib/process/process_win32.c | 105 ++++++++++++++++++++- src/lib/process/process_win32.h | 3 + src/lib/process/subprocess.c | 158 -------------------------------- src/lib/process/subprocess.h | 20 ---- src/test/test_logging.c | 1 - src/test/test_process.c | 40 ++++++++ src/test/test_pt.c | 2 - src/test/test_util.c | 56 +---------- 13 files changed, 171 insertions(+), 240 deletions(-) delete mode 100644 src/lib/process/subprocess.c delete mode 100644 src/lib/process/subprocess.h diff --git a/src/app/config/config.c b/src/app/config/config.c index 90eae50fdd..e33c6b26e8 100644 --- a/src/app/config/config.c +++ b/src/app/config/config.c @@ -142,7 +142,7 @@ #include "lib/process/pidfile.h" #include "lib/process/restrict.h" #include "lib/process/setuid.h" -#include "lib/process/subprocess.h" +#include "lib/process/process.h" #include "lib/net/gethostname.h" #include "lib/thread/numcpus.h" diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index fd4bec7807..0e326f90e9 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -103,7 +103,6 @@ #include "feature/control/control.h" #include "lib/process/process.h" -#include "lib/process/subprocess.h" #include "lib/process/env.h" static smartlist_t * diff --git a/src/lib/process/include.am b/src/lib/process/include.am index aa73356146..a2d54b6238 100644 --- a/src/lib/process/include.am +++ b/src/lib/process/include.am @@ -14,7 +14,6 @@ src_lib_libtor_process_a_SOURCES = \ src/lib/process/process_win32.c \ src/lib/process/restrict.c \ src/lib/process/setuid.c \ - src/lib/process/subprocess.c \ src/lib/process/waitpid.c \ src/lib/process/winprocess_sys.c @@ -32,6 +31,5 @@ noinst_HEADERS += \ src/lib/process/process_win32.h \ src/lib/process/restrict.h \ src/lib/process/setuid.h \ - src/lib/process/subprocess.h \ src/lib/process/waitpid.h \ src/lib/process/winprocess_sys.h diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 915217e132..7275d0a21b 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -26,6 +26,12 @@ /** A list of all process_t instances currently allocated. */ static smartlist_t *processes; +/** + * Boolean. If true, then Tor may call execve or CreateProcess via + * tor_spawn_background. + **/ +static int may_spawn_background_process = 1; + /** Structure to represent a child process. */ struct process_t { /** Process status. */ @@ -114,6 +120,16 @@ process_protocol_to_string(process_protocol_t protocol) /* LCOV_EXCL_STOP */ } +/** + * Turn off may_spawn_background_process, so that all future calls to + * tor_spawn_background are guaranteed to fail. + **/ +void +tor_disable_spawning_background_processes(void) +{ + may_spawn_background_process = 0; +} + /** Initialize the Process subsystem. This function initializes the Process * subsystem's global state. For cleaning up, process_free_all() should * be called. */ @@ -234,6 +250,9 @@ process_exec(process_t *process) { tor_assert(process); + if (BUG(may_spawn_background_process == 0)) + return PROCESS_STATUS_ERROR; + process_status_t status = PROCESS_STATUS_NOT_RUNNING; log_info(LD_PROCESS, "Starting new process: %s", process->command); diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 6092c2da7d..cb5bccbf7b 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -45,6 +45,8 @@ typedef enum { const char *process_protocol_to_string(process_protocol_t protocol); +void tor_disable_spawning_background_processes(void); + struct process_t; typedef struct process_t process_t; diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 7b18903c70..73e19d518f 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -18,13 +18,16 @@ #include "lib/log/win32err.h" #include "lib/process/process.h" #include "lib/process/process_win32.h" -#include "lib/process/subprocess.h" #include "lib/process/env.h" #ifdef HAVE_SYS_TIME_H #include #endif +#ifdef HAVE_STRING_H +#include +#endif + #ifdef _WIN32 /** The size of our intermediate buffers. */ @@ -889,4 +892,104 @@ process_win32_handle_read_completion(process_win32_handle_t *handle, return false; } +/** Format a single argument for being put on a Windows command line. + * Returns a newly allocated string */ +STATIC char * +format_win_cmdline_argument(const char *arg) +{ + char *formatted_arg; + char need_quotes; + const char *c; + int i; + int bs_counter = 0; + /* Backslash we can point to when one is inserted into the string */ + const char backslash = '\\'; + + /* Smartlist of *char */ + smartlist_t *arg_chars; + arg_chars = smartlist_new(); + + /* Quote string if it contains whitespace or is empty */ + need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); + + /* Build up smartlist of *chars */ + for (c=arg; *c != '\0'; c++) { + if ('"' == *c) { + /* Double up backslashes preceding a quote */ + for (i=0; i<(bs_counter*2); i++) + smartlist_add(arg_chars, (void*)&backslash); + bs_counter = 0; + /* Escape the quote */ + smartlist_add(arg_chars, (void*)&backslash); + smartlist_add(arg_chars, (void*)c); + } else if ('\\' == *c) { + /* Count backslashes until we know whether to double up */ + bs_counter++; + } else { + /* Don't double up slashes preceding a non-quote */ + for (i=0; i -#endif -#ifdef HAVE_SYS_PRCTL_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_SIGNAL_H -#include -#endif -#ifdef HAVE_FCNTL_H -#include -#endif -#ifdef HAVE_SYS_WAIT_H -#include -#endif -#include -#include - -/** Format a single argument for being put on a Windows command line. - * Returns a newly allocated string */ -static char * -format_win_cmdline_argument(const char *arg) -{ - char *formatted_arg; - char need_quotes; - const char *c; - int i; - int bs_counter = 0; - /* Backslash we can point to when one is inserted into the string */ - const char backslash = '\\'; - - /* Smartlist of *char */ - smartlist_t *arg_chars; - arg_chars = smartlist_new(); - - /* Quote string if it contains whitespace or is empty */ - need_quotes = (strchr(arg, ' ') || strchr(arg, '\t') || '\0' == arg[0]); - - /* Build up smartlist of *chars */ - for (c=arg; *c != '\0'; c++) { - if ('"' == *c) { - /* Double up backslashes preceding a quote */ - for (i=0; i<(bs_counter*2); i++) - smartlist_add(arg_chars, (void*)&backslash); - bs_counter = 0; - /* Escape the quote */ - smartlist_add(arg_chars, (void*)&backslash); - smartlist_add(arg_chars, (void*)c); - } else if ('\\' == *c) { - /* Count backslashes until we know whether to double up */ - bs_counter++; - } else { - /* Don't double up slashes preceding a non-quote */ - for (i=0; i diff --git a/src/test/test_process.c b/src/test/test_process.c index 2fcb3f6686..3640b86688 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -628,11 +628,51 @@ test_win32(void *arg) process_init(); process_t *process = process_new(""); + char *joined_argv = NULL; /* On Win32 all processes should have a Win32 process handle. */ tt_ptr_op(NULL, OP_NE, process_get_win32_process(process)); + /* Based on some test cases from "Parsing C++ Command-Line Arguments" in + * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline + * will try to only generate simple cases for the child process to parse; + * i.e. we never embed quoted strings in arguments. */ + + const char *argvs[][4] = { + {"a", "bb", "CCC", NULL}, // Normal + {NULL, NULL, NULL, NULL}, // Empty argument list + {"", NULL, NULL, NULL}, // Empty argument + {"\"a", "b\"b", "CCC\"", NULL}, // Quotes + {"a\tbc", "dd dd", "E", NULL}, // Whitespace + {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes + {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote + {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote + { NULL } // Terminator + }; + + const char *cmdlines[] = { + "a bb CCC", + "", + "\"\"", + "\\\"a b\\\"b CCC\\\"", + "\"a\tbc\" \"dd dd\" E", + "a\\\\\\b \"de fg\" H", + "a\\\\\\\"b \\c D\\", + "\"a\\\\b c\" d E", + NULL // Terminator + }; + + int i; + + for (i=0; cmdlines[i]!=NULL; i++) { + log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); + joined_argv = tor_join_win_cmdline(argvs[i]); + tt_str_op(cmdlines[i],OP_EQ, joined_argv); + tor_free(joined_argv); + } + done: + tor_free(joined_argv); process_free(process); process_free_all(); #endif diff --git a/src/test/test_pt.c b/src/test/test_pt.c index c980276b14..b5572ef80c 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -8,7 +8,6 @@ #define UTIL_PRIVATE #define STATEFILE_PRIVATE #define CONTROL_PRIVATE -#define SUBPROCESS_PRIVATE #define PROCESS_PRIVATE #include "core/or/or.h" #include "app/config/config.h" @@ -18,7 +17,6 @@ #include "core/or/circuitbuild.h" #include "app/config/statefile.h" #include "test/test.h" -#include "lib/process/subprocess.h" #include "lib/encoding/confline.h" #include "lib/net/resolve.h" #include "lib/process/process.h" diff --git a/src/test/test_util.c b/src/test/test_util.c index 4fa67b6419..a94153f2dc 100644 --- a/src/test/test_util.c +++ b/src/test/test_util.c @@ -10,7 +10,7 @@ #define UTIL_PRIVATE #define UTIL_MALLOC_PRIVATE #define SOCKET_PRIVATE -#define SUBPROCESS_PRIVATE +#define PROCESS_WIN32_PRIVATE #include "lib/testsupport/testsupport.h" #include "core/or/or.h" #include "lib/container/buffers.h" @@ -22,6 +22,7 @@ #include "test/test.h" #include "lib/memarea/memarea.h" #include "lib/process/waitpid.h" +#include "lib/process/process_win32.h" #include "test/log_test_helpers.h" #include "lib/compress/compress.h" #include "lib/compress/compress_zstd.h" @@ -30,7 +31,6 @@ #include "lib/fs/winlib.h" #include "lib/process/env.h" #include "lib/process/pidfile.h" -#include "lib/process/subprocess.h" #include "lib/intmath/weakrng.h" #include "lib/thread/numcpus.h" #include "lib/math/fp.h" @@ -4395,57 +4395,6 @@ test_util_format_dec_number(void *ptr) return; } -/** - * Test that we can properly format a Windows command line - */ -static void -test_util_join_win_cmdline(void *ptr) -{ - /* Based on some test cases from "Parsing C++ Command-Line Arguments" in - * MSDN but we don't exercise all quoting rules because tor_join_win_cmdline - * will try to only generate simple cases for the child process to parse; - * i.e. we never embed quoted strings in arguments. */ - - const char *argvs[][4] = { - {"a", "bb", "CCC", NULL}, // Normal - {NULL, NULL, NULL, NULL}, // Empty argument list - {"", NULL, NULL, NULL}, // Empty argument - {"\"a", "b\"b", "CCC\"", NULL}, // Quotes - {"a\tbc", "dd dd", "E", NULL}, // Whitespace - {"a\\\\\\b", "de fg", "H", NULL}, // Backslashes - {"a\\\"b", "\\c", "D\\", NULL}, // Backslashes before quote - {"a\\\\b c", "d", "E", NULL}, // Backslashes not before quote - { NULL } // Terminator - }; - - const char *cmdlines[] = { - "a bb CCC", - "", - "\"\"", - "\\\"a b\\\"b CCC\\\"", - "\"a\tbc\" \"dd dd\" E", - "a\\\\\\b \"de fg\" H", - "a\\\\\\\"b \\c D\\", - "\"a\\\\b c\" d E", - NULL // Terminator - }; - - int i; - char *joined_argv = NULL; - - (void)ptr; - - for (i=0; cmdlines[i]!=NULL; i++) { - log_info(LD_GENERAL, "Joining argvs[%d], expecting <%s>", i, cmdlines[i]); - joined_argv = tor_join_win_cmdline(argvs[i]); - tt_str_op(cmdlines[i],OP_EQ, joined_argv); - tor_free(joined_argv); - } - - done: - tor_free(joined_argv); -} - #define MAX_SPLIT_LINE_COUNT 4 struct split_lines_test_t { const char *orig_line; // Line to be split (may contain \0's) @@ -6226,7 +6175,6 @@ struct testcase_t util_tests[] = { UTIL_TEST_WIN_ONLY(load_win_lib, 0), UTIL_TEST(format_hex_number, 0), UTIL_TEST(format_dec_number, 0), - UTIL_TEST(join_win_cmdline, 0), UTIL_TEST(n_bits_set, 0), UTIL_TEST(eat_whitespace, 0), UTIL_TEST(sl_new_from_text_lines, 0), From 6e508e9eb48c88f94cd9431aca6ac5743beb41d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Mon, 26 Nov 2018 07:16:28 +0100 Subject: [PATCH 23/34] Fix tests on kqueue() based platforms. This patch disables fork()'ing of the slow process tests. This fixes the tests on the MacOS and other kqueue() based platforms. Without this patch the main loop exits eearly with EBADF as error. See: https://bugs.torproject.org/28179 --- src/test/test_process_slow.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index b4c5a2d5e2..2ec9ff28a0 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -324,7 +324,7 @@ test_callbacks_terminate(void *arg) } struct testcase_t slow_process_tests[] = { - { "callbacks", test_callbacks, TT_FORK, NULL, NULL }, - { "callbacks_terminate", test_callbacks_terminate, TT_FORK, NULL, NULL }, + { "callbacks", test_callbacks, 0, NULL, NULL }, + { "callbacks_terminate", test_callbacks_terminate, 0, NULL, NULL }, END_OF_TESTCASES }; From f983a60a6c9ce70e1c674458e468b0d7dcd80c01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Tue, 27 Nov 2018 17:36:10 +0100 Subject: [PATCH 24/34] Copy (zlib1|libssp-0).dll to \src\test\ to run test-process.exe. This patch ensures that AppVeyor copies over libssp-0.dll and zlib1.dll to src/test/ to make sure we can run text-process.exe from our slow tests. See: https://bugs.torproject.org/28179 --- .appveyor.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.appveyor.yml b/.appveyor.yml index 837cded43c..63d49536fa 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -73,6 +73,8 @@ test_script: $buildpath = @("C:\msys64\${env:compiler_path}\bin") + $oldpath $env:Path = $buildpath -join ';' Set-Location "${env:build}" + Copy-Item "C:/msys64/${env:compiler_path}/bin/libssp-0.dll" -Destination "${env:build}/src/test" + Copy-Item "C:/msys64/${env:compiler_path}/bin/zlib1.dll" -Destination "${env:build}/src/test" Execute-Bash "VERBOSE=1 make -j2 check" } From bc6983afed24c0b83a49d2cef531dcc036245a04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 28 Nov 2018 18:10:02 +0100 Subject: [PATCH 25/34] Use run_main_loop_until_done() for process_t tests. This patch changes the slow process_t tests to use run_main_loop_until_done() instead of do_main_loop() since do_main_loop() initializes a lot of subsystem callbacks that we don't need to run in our tests. See: https://bugs.torproject.org/28179 --- src/core/mainloop/mainloop.c | 11 +++++------ src/core/mainloop/mainloop.h | 1 + src/test/test_process_slow.c | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/core/mainloop/mainloop.c b/src/core/mainloop/mainloop.c index aaaa5009cb..9e58448f33 100644 --- a/src/core/mainloop/mainloop.c +++ b/src/core/mainloop/mainloop.c @@ -200,7 +200,6 @@ static int can_complete_circuits = 0; #define LAZY_DESCRIPTOR_RETRY_INTERVAL (60) static int conn_close_if_marked(int i); -static int run_main_loop_until_done(void); static void connection_start_reading_from_linked_conn(connection_t *conn); static int connection_should_read_from_linked_conn(connection_t *conn); static void conn_read_callback(evutil_socket_t fd, short event, void *_conn); @@ -2847,10 +2846,6 @@ do_main_loop(void) } } #endif /* defined(HAVE_SYSTEMD_209) */ - - main_loop_should_exit = 0; - main_loop_exit_value = 0; - #ifdef ENABLE_RESTART_DEBUGGING { static int first_time = 1; @@ -2976,10 +2971,14 @@ run_main_loop_once(void) * * Shadow won't invoke this function, so don't fill it up with things. */ -static int +STATIC int run_main_loop_until_done(void) { int loop_result = 1; + + main_loop_should_exit = 0; + main_loop_exit_value = 0; + do { loop_result = run_main_loop_once(); } while (loop_result == 1); diff --git a/src/core/mainloop/mainloop.h b/src/core/mainloop/mainloop.h index 14e80ebb21..730234857d 100644 --- a/src/core/mainloop/mainloop.h +++ b/src/core/mainloop/mainloop.h @@ -100,6 +100,7 @@ extern struct token_bucket_rw_t global_bucket; extern struct token_bucket_rw_t global_relayed_bucket; #ifdef MAINLOOP_PRIVATE +STATIC int run_main_loop_until_done(void); STATIC void close_closeable_connections(void); STATIC void initialize_periodic_events(void); STATIC void teardown_periodic_events(void); diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index 2ec9ff28a0..ad84127bba 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -6,6 +6,7 @@ * \brief Slow test cases for the Process API. */ +#define MAINLOOP_PRIVATE #include "orconfig.h" #include "core/or/or.h" #include "core/mainloop/mainloop.h" @@ -168,7 +169,7 @@ run_main_loop(void) NULL); /* Run our main loop. */ - ret = do_main_loop(); + ret = run_main_loop_until_done(); /* Clean up our main loop timeout timer. */ tt_int_op(ret, OP_EQ, 0); From 22cb3c6ce9164920ff81013d5f8dce3c26911af4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 28 Nov 2018 18:12:30 +0100 Subject: [PATCH 26/34] Call close() on stdin/stdout/stderr in process_terminate(). Call close() on all process handles after we have called kill(pid, SIGTERM). See: https://bugs.torproject.org/28179 --- src/lib/process/process_unix.c | 59 +++++++++++++++++++++++++++++++-- src/lib/process/process_unix.h | 1 + src/lib/process/process_win32.c | 5 +++ 3 files changed, 62 insertions(+), 3 deletions(-) diff --git a/src/lib/process/process_unix.c b/src/lib/process/process_unix.c index 4a9aaa2edd..04d6381aa3 100644 --- a/src/lib/process/process_unix.c +++ b/src/lib/process/process_unix.c @@ -119,7 +119,11 @@ process_unix_free_(process_unix_t *unix_process) if (! unix_process->stderr_handle.reached_eof) process_unix_stop_reading(&unix_process->stderr_handle); - process_unix_stop_writing(&unix_process->stdin_handle); + if (unix_process->stdin_handle.is_writing) + process_unix_stop_writing(&unix_process->stdin_handle); + + /* Close all our file descriptors. */ + process_unix_close_file_descriptors(unix_process); tor_event_free(unix_process->stdout_handle.event); tor_event_free(unix_process->stderr_handle.event); @@ -368,6 +372,8 @@ process_unix_terminate(process_t *process) if (BUG(unix_process->waitpid == NULL)) return false; + bool success = true; + /* Send a SIGTERM to our child process. */ int ret; @@ -376,10 +382,14 @@ process_unix_terminate(process_t *process) if (ret == -1) { log_warn(LD_PROCESS, "Unable to terminate process: %s", strerror(errno)); - return false; + success = false; } - return ret == 0; + /* Close all our FD's. */ + if (! process_unix_close_file_descriptors(unix_process)) + success = false; + + return success; } /** Returns the unique process identifier for the given process. */ @@ -648,4 +658,47 @@ process_unix_read_handle(process_t *process, return ret; } +/** Close the standard in, out, and error handles of the given + * unix_process. */ +STATIC bool +process_unix_close_file_descriptors(process_unix_t *unix_process) +{ + tor_assert(unix_process); + + int ret; + bool success = true; + + if (unix_process->stdin_handle.fd != -1) { + ret = close(unix_process->stdin_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard in"); + success = false; + } + + unix_process->stdin_handle.fd = -1; + } + + if (unix_process->stdout_handle.fd != -1) { + ret = close(unix_process->stdout_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard out"); + success = false; + } + + unix_process->stdout_handle.fd = -1; + } + + if (unix_process->stderr_handle.fd != -1) { + ret = close(unix_process->stderr_handle.fd); + if (ret == -1) { + log_warn(LD_PROCESS, "Unable to close standard error"); + success = false; + } + + unix_process->stderr_handle.fd = -1; + } + + return success; +} + #endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_unix.h b/src/lib/process/process_unix.h index e17c59ea81..86c10d7449 100644 --- a/src/lib/process/process_unix.h +++ b/src/lib/process/process_unix.h @@ -60,6 +60,7 @@ STATIC void process_unix_setup_handle(process_t *process, STATIC int process_unix_read_handle(process_t *, process_unix_handle_t *, buf_t *); +STATIC bool process_unix_close_file_descriptors(process_unix_t *); #endif /* defined(PROCESS_UNIX_PRIVATE). */ #endif /* defined(_WIN32). */ diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 73e19d518f..e75328f3e8 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -293,6 +293,11 @@ process_win32_terminate(process_t *process) return false; } + /* Cleanup our handles. */ + process_win32_cleanup_handle(&win32_process->stdin_handle); + process_win32_cleanup_handle(&win32_process->stdout_handle); + process_win32_cleanup_handle(&win32_process->stderr_handle); + return true; } From 5585cbd08f54f732c32feea276c1a47ec8446c5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 28 Nov 2018 21:55:04 +0100 Subject: [PATCH 27/34] Change the Process exit_callback to return bool. This patch changes our process_t's exit_callback to return a boolean value. If the returned value is true, the process subsystem will call process_free() on the given process_t. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 11 +++++--- src/feature/client/transports.h | 2 +- src/lib/process/process.c | 14 +++++++--- src/lib/process/process.h | 4 +-- src/lib/process/process_win32.c | 48 +++++++++++++++++++++++++-------- src/lib/process/process_win32.h | 2 +- src/test/test_process.c | 5 ++-- src/test/test_process_slow.c | 5 ++-- 8 files changed, 65 insertions(+), 26 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 0e326f90e9..e3cc679411 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1759,8 +1759,9 @@ managed_proxy_stderr_callback(process_t *process, char *line, size_t size) /** Callback function that is called when our PT process terminates. The * process exit code can be found in exit_code and our process can be - * found in process. */ -STATIC void + * found in process. Returns true iff we want the process subsystem to + * free our process_t handle for us. */ +STATIC bool managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) { tor_assert(process); @@ -1772,10 +1773,14 @@ managed_proxy_exit_callback(process_t *process, process_exit_code_t exit_code) /* We detach ourself from the MP (if we are attached) and free ourself. */ managed_proxy_t *mp = process_get_data(process); + /* If we are still attached to the process, it is probably because our PT + * process crashed before we got to call process_set_data(p, NULL); */ if (BUG(mp != NULL)) { + /* FIXME(ahf): Our process stopped without us having told it to stop + * (crashed). Should we restart it here? */ mp->process = NULL; process_set_data(process, NULL); } - process_free(process); + return true; } diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index fbb720aac6..ba8cbf7105 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -145,7 +145,7 @@ STATIC void free_execve_args(char **arg); STATIC void managed_proxy_stdout_callback(process_t *, char *, size_t); STATIC void managed_proxy_stderr_callback(process_t *, char *, size_t); -STATIC void managed_proxy_exit_callback(process_t *, process_exit_code_t); +STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); #endif /* defined(PT_PRIVATE) */ diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 7275d0a21b..75bffe35b9 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -612,7 +612,8 @@ process_notify_event_stdin(process_t *process) /** This function is called by the Process backend when a given process have * terminated. The exit status code is passed in exit_code. We mark the * process as no longer running and calls the exit_callback with - * information about the process termination. */ + * information about the process termination. The given process is + * free'd iff the exit_callback returns true. */ void process_notify_event_exit(process_t *process, process_exit_code_t exit_code) { @@ -626,9 +627,14 @@ process_notify_event_exit(process_t *process, process_exit_code_t exit_code) process->exit_code = exit_code; /* Call our exit callback, if it exists. */ - if (process->exit_callback) { - process->exit_callback(process, exit_code); - } + bool free_process_handle = false; + + /* The exit callback will tell us if we should process_free() our handle. */ + if (process->exit_callback) + free_process_handle = process->exit_callback(process, exit_code); + + if (free_process_handle) + process_free(process); } /** This function is called whenever the Process backend have notified us that diff --git a/src/lib/process/process.h b/src/lib/process/process.h index cb5bccbf7b..4b0fae4250 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -56,8 +56,8 @@ typedef uint64_t process_pid_t; typedef void (*process_read_callback_t)(process_t *, char *, size_t); -typedef void (*process_exit_callback_t)(process_t *, - process_exit_code_t); +typedef bool +(*process_exit_callback_t)(process_t *, process_exit_code_t); void process_init(void); void process_free_all(void); diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index e75328f3e8..911bad3933 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -474,28 +474,47 @@ process_win32_timer_callback(periodic_timer_t *timer, void *data) tor_assert(timer == periodic_timer); tor_assert(data == NULL); - log_debug(LD_PROCESS, "Windows Process I/O timer ticked"); - /* Move the process into an alertable state. */ process_win32_trigger_completion_callbacks(); /* Check if our processes are still alive. */ - const smartlist_t *processes = process_get_all_processes(); - SMARTLIST_FOREACH(processes, process_t *, p, - process_win32_timer_test_process(p)); + /* Since the call to process_win32_timer_test_process() might call + * process_notify_event_exit() which again might call process_free() which + * updates the list of processes returned by process_get_all_processes() it + * is important here that we make sure to not touch the list of processes if + * the call to process_win32_timer_test_process() returns true. */ + bool done; + + do { + const smartlist_t *processes = process_get_all_processes(); + done = true; + + SMARTLIST_FOREACH_BEGIN(processes, process_t *, process) { + /* If process_win32_timer_test_process() returns true, it means that + * smartlist_remove() might have been called on the list returned by + * process_get_all_processes(). We start the loop over again until we + * have a succesful run over the entire list where the list was not + * modified. */ + if (process_win32_timer_test_process(process)) { + done = false; + break; + } + } SMARTLIST_FOREACH_END(process); + } while (! done); } /** Test whether a given process is still alive. Notify the Process subsystem - * if our process have died. */ -STATIC void + * if our process have died. Returns true iff the given process have + * terminated. */ +STATIC bool process_win32_timer_test_process(process_t *process) { tor_assert(process); /* No need to look at processes that don't claim they are running. */ if (process_get_status(process) != PROCESS_STATUS_RUNNING) - return; + return false; process_win32_t *win32_process = process_get_win32_process(process); BOOL ret = FALSE; @@ -508,12 +527,19 @@ process_win32_timer_test_process(process_t *process) if (! ret) { log_warn(LD_PROCESS, "GetExitCodeProcess() failed: %s", format_win32_error(GetLastError())); - return; + return false; } - /* Notify our process_t that our process have terminated. */ - if (exit_code != STILL_ACTIVE) + /* Notify our process_t that our process have terminated. Since our + * exit_callback might decide to process_free() our process handle it is very + * important that we do not touch the process_t after the call to + * process_notify_event_exit(). */ + if (exit_code != STILL_ACTIVE) { process_notify_event_exit(process, exit_code); + return true; + } + + return false; } /** Create a new overlapped named pipe. This function creates a new connected, diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index 3c809dfd23..00de8c949b 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -49,7 +49,7 @@ STATIC void process_win32_timer_start(void); STATIC void process_win32_timer_stop(void); STATIC bool process_win32_timer_running(void); STATIC void process_win32_timer_callback(periodic_timer_t *, void *); -STATIC void process_win32_timer_test_process(process_t *); +STATIC bool process_win32_timer_test_process(process_t *); /* I/O pipe handling. */ struct process_win32_handle_t; diff --git a/src/test/test_process.c b/src/test/test_process.c index 3640b86688..17481097be 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -125,7 +125,7 @@ process_stderr_callback(process_t *process, char *data, size_t size) return; } -static void +static bool process_exit_callback(process_t *process, process_exit_code_t exit_code) { tt_ptr_op(process, OP_NE, NULL); @@ -134,7 +134,8 @@ process_exit_callback(process_t *process, process_exit_code_t exit_code) process_data->exit_code = exit_code; done: - return; + /* Do not free up our process_t. */ + return false; } static void diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index ad84127bba..b23492e972 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -94,7 +94,7 @@ process_stderr_callback(process_t *process, char *data, size_t size) return; } -static void +static bool process_exit_callback(process_t *process, process_exit_code_t exit_code) { tt_ptr_op(process, OP_NE, NULL); @@ -106,7 +106,8 @@ process_exit_callback(process_t *process, process_exit_code_t exit_code) tor_shutdown_event_loop_and_exit(0); done: - return; + /* Do not free up our process_t. */ + return false; } #ifdef _WIN32 From ec2ae3ed8b542950dc774223607f14f524375a50 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 29 Nov 2018 16:42:56 +0100 Subject: [PATCH 28/34] Change EVENT_TRANSPORT_LOG to EVENT_PT_LOG. This patch changes our EVENT_TRANSPORT_LOG event to be EVENT_PT_LOG. The new message includes the path to the PT executable instead of the transport name, since one PT binary can include multiple transport they sometimes might need to log messages that are not specific to a given transport. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 33 ++++++++++----------------------- src/feature/client/transports.h | 2 +- src/feature/control/control.c | 10 +++++----- src/feature/control/control.h | 5 ++--- src/test/test_pt.c | 6 +++--- 5 files changed, 21 insertions(+), 35 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index e3cc679411..df7991846c 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -910,7 +910,7 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; } else if (!strcmpstart(line, PROTO_LOG)) { - parse_log_line(line); + parse_log_line(line, mp); return; } @@ -1135,40 +1135,27 @@ parse_proxy_error(const char *line) /** Parses a LOG line and emit log events accordingly. */ STATIC void -parse_log_line(const char *line) +parse_log_line(const char *line, managed_proxy_t *mp) { - smartlist_t *items = smartlist_new(); + tor_assert(line); + tor_assert(mp); if (strlen(line) < (strlen(PROTO_LOG) + 1)) { log_warn(LD_PT, "Managed proxy sent us a %s line " - "with missing arguments.", PROTO_LOG); + "with missing argument.", PROTO_LOG); goto done; } - const char *arguments = line + strlen(PROTO_LOG) + 1; + const char *message = line + strlen(PROTO_LOG) + 1; - /* The format is 'LOG '. We accept empty messages. */ - smartlist_split_string(items, arguments, NULL, - SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, 2); - - if (smartlist_len(items) < 2) { - log_warn(LD_PT, "Managed proxy sent us a %s line " - "with too few arguments.", PROTO_LOG); - goto done; - } - - const char *transport_name = smartlist_get(items, 0); - const char *message = smartlist_get(items, 1); - - log_info(LD_PT, "Managed proxy transport \"%s\" says: %s", - transport_name, message); + log_info(LD_PT, "Managed proxy \"%s\" says: %s", + mp->argv[0], message); /* Emit control port event. */ - control_event_transport_log(transport_name, message); + control_event_pt_log(mp->argv[0], message); done: - SMARTLIST_FOREACH(items, char *, s, tor_free(s)); - smartlist_free(items); + return; } /** Return a newly allocated string that tor should place in diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index ba8cbf7105..88735a7211 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -128,7 +128,7 @@ STATIC int parse_version(const char *line, managed_proxy_t *mp); STATIC void parse_env_error(const char *line); STATIC void parse_proxy_error(const char *line); STATIC void handle_proxy_line(const char *line, managed_proxy_t *mp); -STATIC void parse_log_line(const char *line); +STATIC void parse_log_line(const char *line, managed_proxy_t *mp); STATIC char *get_transport_options_for_server_proxy(const managed_proxy_t *mp); STATIC void managed_proxy_destroy(managed_proxy_t *mp, diff --git a/src/feature/control/control.c b/src/feature/control/control.c index b6505a85d6..ebda51781a 100644 --- a/src/feature/control/control.c +++ b/src/feature/control/control.c @@ -7395,14 +7395,14 @@ control_event_transport_launched(const char *mode, const char *transport_name, mode, transport_name, fmt_addr(addr), port); } -/** A pluggable transport called transport_name has emitted a log +/** A pluggable transport called pt_name has emitted a log * message found in message. */ void -control_event_transport_log(const char *transport_name, const char *message) +control_event_pt_log(const char *pt_name, const char *message) { - send_control_event(EVENT_TRANSPORT_LOG, - "650 TRANSPORT_LOG %s %s\r\n", - transport_name, + send_control_event(EVENT_PT_LOG, + "650 PT_LOG %s %s\r\n", + pt_name, message); } diff --git a/src/feature/control/control.h b/src/feature/control/control.h index eb2b5676ea..c554c32539 100644 --- a/src/feature/control/control.h +++ b/src/feature/control/control.h @@ -205,8 +205,7 @@ void control_event_clients_seen(const char *controller_str); void control_event_transport_launched(const char *mode, const char *transport_name, tor_addr_t *addr, uint16_t port); -void control_event_transport_log(const char *transport_name, - const char *message); +void control_event_pt_log(const char *pt_name, const char *message); const char *rend_auth_type_to_string(rend_auth_type_t auth_type); MOCK_DECL(const char *, node_describe_longname_by_id,(const char *id_digest)); void control_event_hs_descriptor_requested(const char *onion_address, @@ -295,7 +294,7 @@ void control_free_all(void); #define EVENT_HS_DESC 0x0021 #define EVENT_HS_DESC_CONTENT 0x0022 #define EVENT_NETWORK_LIVENESS 0x0023 -#define EVENT_TRANSPORT_LOG 0x0024 +#define EVENT_PT_LOG 0x0024 #define EVENT_MAX_ 0x0024 /* sizeof(control_connection_t.event_mask) in bits, currently a uint64_t */ diff --git a/src/test/test_pt.c b/src/test/test_pt.c index b5572ef80c..8fcdd5c1e8 100644 --- a/src/test/test_pt.c +++ b/src/test/test_pt.c @@ -310,7 +310,7 @@ process_read_stdout_replacement(process_t *process, buf_t *buffer) } else if (times_called <= 6) { buf_add_string(buffer, "SMETHODS DONE\n"); } else if (times_called <= 7) { - buf_add_string(buffer, "LOG mock3 Oh noes, something bad happened. " + buf_add_string(buffer, "LOG Oh noes, something bad happened. " "What do we do!?\n"); } @@ -417,10 +417,10 @@ test_pt_configure_proxy(void *arg) process_notify_event_stdout(mp->process); tt_int_op(controlevent_n, OP_EQ, 6); - tt_int_op(controlevent_event, OP_EQ, EVENT_TRANSPORT_LOG); + tt_int_op(controlevent_event, OP_EQ, EVENT_PT_LOG); tt_int_op(smartlist_len(controlevent_msgs), OP_EQ, 6); tt_str_op(smartlist_get(controlevent_msgs, 5), OP_EQ, - "650 TRANSPORT_LOG mock3 Oh noes, something bad happened. " + "650 PT_LOG Oh noes, something bad happened. " "What do we do!?\r\n"); { /* check that the transport info were saved properly in the tor state */ From cacdd290872420d51f880d75466f6d6e22430466 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 7 Dec 2018 02:09:22 +0100 Subject: [PATCH 29/34] Use `const char *` instead of `char *` for line parameter for process callbacks. This patch changes the type definition of the process callbacks to use `const char *` instead of `char *`. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 8 ++++++-- src/feature/client/transports.h | 4 ++-- src/lib/process/process.h | 2 +- src/test/test_process.c | 4 ++-- src/test/test_process_slow.c | 4 ++-- 5 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index df7991846c..0f456ba1e6 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -1711,7 +1711,9 @@ tor_escape_str_for_pt_args(const char *string, const char *chars_to_escape) * stdout. Our process can be found in process, the data can be found in * line and the length of our line is given in size. */ STATIC void -managed_proxy_stdout_callback(process_t *process, char *line, size_t size) +managed_proxy_stdout_callback(process_t *process, + const char *line, + size_t size) { tor_assert(process); tor_assert(line); @@ -1732,7 +1734,9 @@ managed_proxy_stdout_callback(process_t *process, char *line, size_t size) * stderr. Our process can be found in process, the data can be found in * line and the length of our line is given in size. */ STATIC void -managed_proxy_stderr_callback(process_t *process, char *line, size_t size) +managed_proxy_stderr_callback(process_t *process, + const char *line, + size_t size) { tor_assert(process); tor_assert(line); diff --git a/src/feature/client/transports.h b/src/feature/client/transports.h index 88735a7211..a3994a0099 100644 --- a/src/feature/client/transports.h +++ b/src/feature/client/transports.h @@ -143,8 +143,8 @@ STATIC char* get_pt_proxy_uri(void); STATIC void free_execve_args(char **arg); -STATIC void managed_proxy_stdout_callback(process_t *, char *, size_t); -STATIC void managed_proxy_stderr_callback(process_t *, char *, size_t); +STATIC void managed_proxy_stdout_callback(process_t *, const char *, size_t); +STATIC void managed_proxy_stderr_callback(process_t *, const char *, size_t); STATIC bool managed_proxy_exit_callback(process_t *, process_exit_code_t); #endif /* defined(PT_PRIVATE) */ diff --git a/src/lib/process/process.h b/src/lib/process/process.h index 4b0fae4250..179db19aeb 100644 --- a/src/lib/process/process.h +++ b/src/lib/process/process.h @@ -54,7 +54,7 @@ typedef uint64_t process_exit_code_t; typedef uint64_t process_pid_t; typedef void (*process_read_callback_t)(process_t *, - char *, + const char *, size_t); typedef bool (*process_exit_callback_t)(process_t *, process_exit_code_t); diff --git a/src/test/test_process.c b/src/test/test_process.c index 17481097be..9b62862f04 100644 --- a/src/test/test_process.c +++ b/src/test/test_process.c @@ -98,7 +98,7 @@ process_mocked_write_stdin(process_t *process, buf_t *buffer) } static void -process_stdout_callback(process_t *process, char *data, size_t size) +process_stdout_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); @@ -112,7 +112,7 @@ process_stdout_callback(process_t *process, char *data, size_t size) } static void -process_stderr_callback(process_t *process, char *data, size_t size) +process_stderr_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); diff --git a/src/test/test_process_slow.c b/src/test/test_process_slow.c index b23492e972..cc7e8c5a9c 100644 --- a/src/test/test_process_slow.c +++ b/src/test/test_process_slow.c @@ -67,7 +67,7 @@ process_data_free(process_data_t *process_data) } static void -process_stdout_callback(process_t *process, char *data, size_t size) +process_stdout_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); @@ -81,7 +81,7 @@ process_stdout_callback(process_t *process, char *data, size_t size) } static void -process_stderr_callback(process_t *process, char *data, size_t size) +process_stderr_callback(process_t *process, const char *data, size_t size) { tt_ptr_op(process, OP_NE, NULL); tt_ptr_op(data, OP_NE, NULL); From 0d796cce17bdda605f5e768971935d7f876e33fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Wed, 12 Dec 2018 18:48:45 +0100 Subject: [PATCH 30/34] Use errno directly if we are not reading/writing from/to a socket. See: https://bugs.torproject.org/28179 --- src/lib/net/buffers_net.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/lib/net/buffers_net.c b/src/lib/net/buffers_net.c index da7043d5cb..9430a6b8c4 100644 --- a/src/lib/net/buffers_net.c +++ b/src/lib/net/buffers_net.c @@ -51,7 +51,8 @@ read_to_chunk(buf_t *buf, chunk_t *chunk, tor_socket_t fd, size_t at_most, read_result = read(fd, CHUNK_WRITE_PTR(chunk), at_most); if (read_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) @@ -152,7 +153,8 @@ flush_chunk(tor_socket_t fd, buf_t *buf, chunk_t *chunk, size_t sz, write_result = write(fd, chunk->data, sz); if (write_result < 0) { - int e = tor_socket_errno(fd); + int e = is_socket ? tor_socket_errno(fd) : errno; + if (!ERRNO_IS_EAGAIN(e)) { /* it's a real error */ #ifdef _WIN32 if (e == WSAENOBUFS) From a33a77d9cd3c06b5a871e99631b7f1c40bed23c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 13 Dec 2018 00:48:33 +0100 Subject: [PATCH 31/34] Document the format of process_t::arguments. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index 75bffe35b9..fb76a0a725 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -55,7 +55,11 @@ struct process_t { /** Name of the command we want to execute (for example: /bin/ls). */ char *command; - /** The arguments used for the new process. */ + /** The arguments used for the new process. The format here is one argument + * per element of the smartlist_t. On Windows these arguments are combined + * together using the tor_join_win_cmdline function. On Unix the + * process name (argv[0]) and the trailing NULL is added automatically before + * the process is executed. */ smartlist_t *arguments; /** The environment used for the new process. */ From fab22509d70ff3289f7300788bd9243659268f69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Thu, 13 Dec 2018 00:52:39 +0100 Subject: [PATCH 32/34] Make Windows process event timer API available for dormant interface. This patch changes the API of the Windows backend of the Process subsystem to allow the dormant interface to disable the Process event timer. See: https://bugs.torproject.org/28179 --- src/lib/process/process_win32.c | 6 +++--- src/lib/process/process_win32.h | 9 +++++---- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/lib/process/process_win32.c b/src/lib/process/process_win32.c index 911bad3933..e9367dafc9 100644 --- a/src/lib/process/process_win32.c +++ b/src/lib/process/process_win32.c @@ -429,7 +429,7 @@ process_win32_trigger_completion_callbacks(void) /** Start the periodic timer which is reponsible for checking whether processes * are still alive and to make sure that the Tor process is periodically being * moved into an alertable state. */ -STATIC void +void process_win32_timer_start(void) { /* Make sure we never start our timer if it's already running. */ @@ -447,7 +447,7 @@ process_win32_timer_start(void) } /** Stops the periodic timer. */ -STATIC void +void process_win32_timer_stop(void) { if (BUG(periodic_timer == NULL)) @@ -458,7 +458,7 @@ process_win32_timer_stop(void) } /** Returns true iff the periodic timer is running. */ -STATIC bool +bool process_win32_timer_running(void) { return periodic_timer != NULL; diff --git a/src/lib/process/process_win32.h b/src/lib/process/process_win32.h index 00de8c949b..8ab4880fbd 100644 --- a/src/lib/process/process_win32.h +++ b/src/lib/process/process_win32.h @@ -43,11 +43,12 @@ int process_win32_read_stderr(struct process_t *process, buf_t *buffer); void process_win32_trigger_completion_callbacks(void); -#ifdef PROCESS_WIN32_PRIVATE /* Timer handling. */ -STATIC void process_win32_timer_start(void); -STATIC void process_win32_timer_stop(void); -STATIC bool process_win32_timer_running(void); +void process_win32_timer_start(void); +void process_win32_timer_stop(void); +bool process_win32_timer_running(void); + +#ifdef PROCESS_WIN32_PRIVATE STATIC void process_win32_timer_callback(periodic_timer_t *, void *); STATIC bool process_win32_timer_test_process(process_t *); From 651cdd05b7071bff4da7c3335f697d90c82c9f3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 14 Dec 2018 02:17:00 +0100 Subject: [PATCH 33/34] Add an additional space when we check for the PROTO_LOG handler. See: https://bugs.torproject.org/28179 --- src/feature/client/transports.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/feature/client/transports.c b/src/feature/client/transports.c index 0f456ba1e6..de53fe3469 100644 --- a/src/feature/client/transports.c +++ b/src/feature/client/transports.c @@ -909,7 +909,11 @@ handle_proxy_line(const char *line, managed_proxy_t *mp) parse_proxy_error(line); goto err; - } else if (!strcmpstart(line, PROTO_LOG)) { + + /* We check for the additional " " after the PROTO_LOG string to make sure + * we can later extend this big if/else-if table with something that begins + * with "LOG" without having to get the order right. */ + } else if (!strcmpstart(line, PROTO_LOG " ")) { parse_log_line(line, mp); return; } From c8b8b15f0eb2651dea694a057e70e6b8c34dbe05 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alexander=20F=C3=A6r=C3=B8y?= Date: Fri, 14 Dec 2018 03:31:56 +0100 Subject: [PATCH 34/34] Ensure that line_size >= 1 before trying to trim input string. See: https://bugs.torproject.org/28179 --- src/lib/process/process.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/lib/process/process.c b/src/lib/process/process.c index fb76a0a725..b3370e919f 100644 --- a/src/lib/process/process.c +++ b/src/lib/process/process.c @@ -766,13 +766,13 @@ process_read_lines(process_t *process, tor_assert(ret != -1); /* Remove \n from the end of the line. */ - if (data[line_size - 1] == '\n') { + if (line_size >= 1 && data[line_size - 1] == '\n') { data[line_size - 1] = '\0'; --line_size; } /* Remove \r from the end of the line. */ - if (data[line_size - 1] == '\r') { + if (line_size >= 1 && data[line_size - 1] == '\r') { data[line_size - 1] = '\0'; --line_size; }