Flush more changes from sandbox

- make clients cache directories and use them to seed their router lists
   at startup.  This means clients have a datadir again.
 - Introduce a global_write_bucket.  We need to respond better to exhausting
   it.
 - Remove the last vestiges of LinkPadding and TrafficShaping.
 - Configuration infrastructure support for warning on obsolete options.
 - Refactor directory header parsing to use smartlist_split_string.
 - Respond to content-encoding headers by trying to uncompress as appropriate.
 - Reply with a deflated directory when a client asks for "dir.z".
   (We could use allow-encodings instead, but allow-encodings isn't
   specified in HTTP 1.0.)


svn:r2335
This commit is contained in:
Nick Mathewson 2004-09-08 06:52:33 +00:00
parent b6798866d0
commit c66e4c4870
8 changed files with 174 additions and 78 deletions

View File

@ -12,32 +12,46 @@ ARMA - arma claims
X Abandoned
0.0.9:
- the user interface interface
- let tor clients use http proxies for dir fetching
- let tor servers use http proxies for port 80 exits
- write instructions for port-forwarding directives or programs
o Fix OutboundBindAddress
o Config defaults should be consistant with config file and no
config file.
o write instructions for port-forwarding directives or programs
to let people run on ports 80 and 443 without needing to bind
tor to them.
o clean up all the comma-separated stuff (eg exit policies) into
smartlists.
o investigate sctp for alternate transport.
o Document all undocumented options, or mark them as undocumented
in the source.
o bandwidth buckets for write as well as read.
. Cached-directory changes:
o make clients store the cached-directory to disk,
o and use it when they startup, so they don't need to bootstrap
from the authdirservers every time they start.
- also, once we've reduced authdirserver entries to config
lines, we can have lines that list cacheddirservers too.
. compress the directory.
o Implement gzip/zlib wrappers
o Compress directories as they're cached/generated
o When requested, give a compressed directory.
o Decompress incoming HTTP based on Content-Encoding
- Once dirservers are running new code, make clients
request compressed directories. (Alternative: Switch
to HTTP/1.1 and use Allowed-Encoding. Is that really
what we want?)
N - switch dirservers entries to config lines.
N - let tor clients use http proxies for dir fetching
N - per-month byte allowances.
Nr - figure out how to handle rendezvousing with unverified nodes.
Nr - figure out enclaves, e.g. so we know what to recommend that people
do, and so running a tor server on your website is helpful.
- node 'groups' that are known to be in the same zone of control.
- let tor servers use http proxies for port 80 exits
- the user interface interface
- add ipv6 support.
- learn from ben about his openssl-reinitialization-trick to
rotate tls keys without making new connections.
- figure out how to handle rendezvousing with unverified nodes.
- clean up all the comma-separated stuff (eg exit policies) into
smartlists.
- per-month byte allowances.
- node 'groups' that are known to be in the same zone of control.
- figure out enclaves, e.g. so we know what to recommend that people
do, and so running a tor server on your website is helpful.
- compress the directory.
- switch dirservers entries to config lines.
- investigate sctp for alternate transport.
- nt services on win32.
- bandwidth buckets for write as well as read.
- make clients store the cached-directory to disk, and use it
when they startup, so they don't need to bootstrap from the
authdirservers every time they start. also, once we've reduced
authdirserver entries to config lines, we can have lines that
list cacheddirservers too.
- add ipv6 support.
D nt services on win32.
0.0.8:
- fix sprintf's to snprintf's?
@ -94,6 +108,7 @@ NICK . unify similar config entries that need to be split. put them
- if destination IP is running a tor node, extend a circuit there
before sending begin.
* don't do this for now. figure out how enclaves work. but do enclaves soon.
- Track max ten-second b/w ever seen, to show operator
more features, complex:
- compress the directory. client sends http header

View File

@ -23,6 +23,7 @@ typedef enum config_type_t {
CONFIG_TYPE_CSV, /**< A list of strings, separated by commas and optional
* whitespace. */
CONFIG_TYPE_LINELIST, /**< Uninterpreted config lines */
CONFIG_TYPE_OBSOLETE, /**< Obsolete (ignored) option. */
} config_type_t;
/** Largest allowed config line */
@ -170,6 +171,9 @@ static int config_compare(struct config_line_t *c, const char *key, config_type_
*(struct config_line_t**)arg =
config_line_prepend(*(struct config_line_t**)arg, c->key, c->value);
break;
case CONFIG_TYPE_OBSOLETE:
log_fn(LOG_WARN, "Skipping obsolete configuration option '%s'", c->key);
break;
}
return 1;
}
@ -225,7 +229,7 @@ static int config_assign(or_options_t *options, struct config_line_t *list) {
config_compare(list, "LogLevel", CONFIG_TYPE_LINELIST, &options->LogOptions) ||
config_compare(list, "LogFile", CONFIG_TYPE_LINELIST, &options->LogOptions) ||
config_compare(list, "LinkPadding", CONFIG_TYPE_BOOL, &options->LinkPadding) ||
config_compare(list, "LinkPadding", CONFIG_TYPE_OBSOLETE, NULL) ||
config_compare(list, "MaxConn", CONFIG_TYPE_INT, &options->MaxConn) ||
config_compare(list, "MaxOnionsPending",CONFIG_TYPE_INT, &options->MaxOnionsPending) ||
@ -252,7 +256,7 @@ static int config_assign(or_options_t *options, struct config_line_t *list) {
config_compare(list, "SocksBindAddress",CONFIG_TYPE_LINELIST,&options->SocksBindAddress) ||
config_compare(list, "SocksPolicy", CONFIG_TYPE_LINELIST,&options->SocksPolicy) ||
config_compare(list, "TrafficShaping", CONFIG_TYPE_BOOL, &options->TrafficShaping) ||
config_compare(list, "TrafficShaping", CONFIG_TYPE_OBSOLETE, NULL) ||
config_compare(list, "User", CONFIG_TYPE_STRING, &options->User)
@ -1024,7 +1028,7 @@ const char *get_data_directory(or_options_t *options) {
const char *d;
if (options->DataDirectory)
d = options->DataDirectory;
else if (server_mode()) {
else {
#ifdef MS_WINDOWS
char *p;
p = tor_malloc(MAX_PATH);
@ -1037,10 +1041,7 @@ const char *get_data_directory(or_options_t *options) {
#else
d = "~/.tor";
#endif
} else
d = NULL; /* XXX008 don't create datadir until we have something
we'll be putting in it */
}
if (d && strncmp(d,"~/",2)==0) {
char *fn = expand_filename(d);
tor_free(options->DataDirectory);

View File

@ -592,26 +592,22 @@ int retry_all_listeners(void) {
return 0;
}
extern int global_read_bucket;
extern int global_read_bucket, global_write_bucket;
/** How many bytes at most can we read onto this connection? */
int connection_bucket_read_limit(connection_t *conn) {
int at_most;
if(options.LinkPadding) {
at_most = global_read_bucket;
/* do a rudimentary round-robin so one circuit can't hog a connection */
if(connection_speaks_cells(conn)) {
at_most = 32*(CELL_NETWORK_SIZE);
} else {
/* do a rudimentary round-robin so one circuit can't hog a connection */
if(connection_speaks_cells(conn)) {
at_most = 32*(CELL_NETWORK_SIZE);
} else {
at_most = 32*(RELAY_PAYLOAD_SIZE);
}
if(at_most > global_read_bucket)
at_most = global_read_bucket;
at_most = 32*(RELAY_PAYLOAD_SIZE);
}
if(at_most > global_read_bucket)
at_most = global_read_bucket;
if(connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN)
if(at_most > conn->receiver_bucket)
at_most = conn->receiver_bucket;
@ -620,7 +616,7 @@ int connection_bucket_read_limit(connection_t *conn) {
}
/** We just read num_read onto conn. Decrement buckets appropriately. */
void connection_bucket_decrement(connection_t *conn, int num_read) {
static void connection_read_bucket_decrement(connection_t *conn, int num_read) {
global_read_bucket -= num_read; tor_assert(global_read_bucket >= 0);
if(connection_speaks_cells(conn) && conn->state == OR_CONN_STATE_OPEN) {
conn->receiver_bucket -= num_read; tor_assert(conn->receiver_bucket >= 0);
@ -648,6 +644,7 @@ static struct timeval current_time;
void connection_bucket_init(void) {
tor_gettimeofday(&current_time);
global_read_bucket = options.BandwidthBurst; /* start it at max traffic */
global_write_bucket = options.BandwidthBurst; /* start it at max traffic */
}
/** Some time has passed; increment buckets appropriately. */
@ -662,11 +659,15 @@ void connection_bucket_refill(struct timeval *now) {
current_time.tv_sec = now->tv_sec; /* update current_time */
/* (ignore usecs for now) */
/* refill the global bucket */
/* refill the global buckets */
if(global_read_bucket < options.BandwidthBurst) {
global_read_bucket += options.BandwidthRate;
log_fn(LOG_DEBUG,"global_read_bucket now %d.", global_read_bucket);
}
if(global_write_bucket < options.BandwidthBurst) {
global_write_bucket += options.BandwidthRate;
log_fn(LOG_DEBUG,"global_write_bucket now %d.", global_write_bucket);
}
/* refill the per-connection buckets */
get_connection_array(&carray,&n);
@ -680,6 +681,8 @@ void connection_bucket_refill(struct timeval *now) {
if(conn->wants_to_read == 1 /* it's marked to turn reading back on now */
&& global_read_bucket > 0 /* and we're allowed to read */
&& global_write_bucket > 0 /* and we're allowed to write (XXXX,
* not the best place to check this.) */
&& (!connection_speaks_cells(conn) ||
conn->state != OR_CONN_STATE_OPEN ||
conn->receiver_bucket > 0)) {
@ -815,7 +818,7 @@ static int connection_read_to_buf(connection_t *conn) {
rep_hist_note_bytes_read(result, time(NULL));
}
connection_bucket_decrement(conn, result);
connection_read_bucket_decrement(conn, result);
return 0;
}
@ -947,6 +950,8 @@ int connection_handle_write(connection_t *conn) {
rep_hist_note_bytes_written(result, now);
}
global_write_bucket -= result;
if(!connection_wants_to_flush(conn)) { /* it's done flushing */
if(connection_finished_flushing(conn) < 0) {
/* already marked */

View File

@ -210,6 +210,7 @@ directory_initiate_command(routerinfo_t *router, uint8_t purpose,
static void directory_send_command(connection_t *conn, int purpose,
const char *payload, int payload_len) {
char fetchwholedir[] = "GET / HTTP/1.0\r\n\r\n";
char fetchwholedir_z[] = "GET /dir.z HTTP/1.0\r\n\r\n";
char fetchrunninglist[] = "GET /running-routers HTTP/1.0\r\n\r\n";
char tmp[8192];
@ -288,11 +289,12 @@ parse_http_url(char *headers, char **url)
* Otherwise, return -1.
*/
static int
parse_http_response(char *headers, int *code, char **message, time_t *date)
parse_http_response(char *headers, int *code, char **message, time_t *date,
int *compression)
{
int n1, n2;
const char *cp;
char datestr[RFC1123_TIME_LEN+1];
smartlist_t *parsed_headers;
tor_assert(headers && code);
while(isspace((int)*headers)) headers++; /* tolerate leading whitespace */
@ -307,22 +309,40 @@ parse_http_response(char *headers, int *code, char **message, time_t *date)
if(message) {
/* XXX should set *message correctly */
}
parsed_headers = smartlist_create();
smartlist_split_string(parsed_headers, headers, "\n",
SPLIT_SKIP_SPACE|SPLIT_IGNORE_BLANK, -1);
if (date) {
cp = headers;
*date = 0;
while (cp && (cp = strchr(cp, '\n'))) {
++cp;
strlcpy(datestr, cp, 7);
if (strcmpstart(cp, "Date: ") == 0) {
strlcpy(datestr, cp+6, sizeof(datestr));
SMARTLIST_FOREACH(parsed_headers, const char *, s,
if (!strcmpstart(s, "Date: ")) {
strlcpy(datestr, s+6, sizeof(datestr));
/* This will do nothing on failure, so we don't need to check
the result. We shouldn't warn, since there are many other valid
date formats besides the one we use. */
parse_rfc1123_time(datestr, date);
break;
}
});
}
if (compression) {
const char *enc = NULL;
SMARTLIST_FOREACH(parsed_headers, const char *, s,
if (!strcmpstart(s, "Content-Encoding: ")) {
enc = s+16; break;
});
if (!enc || strcmp(enc, "identity")) {
*compression = 0;
} else if (!strcmp(enc, "deflate") || !strcmp(enc, "x-deflate")) {
*compression = ZLIB_METHOD;
} else if (!strcmp(enc, "gzip") || !strcmp(enc, "x-gzip")) {
*compression = GZIP_METHOD;
} else {
log_fn(LOG_WARN, "Unrecognized content encoding: '%s'", enc);
*compression = 0;
}
}
SMARTLIST_FOREACH(parsed_headers, char *, s, tor_free(s));
smartlist_free(parsed_headers);
return 0;
}
@ -342,6 +362,7 @@ connection_dir_client_reached_eof(connection_t *conn)
int status_code;
time_t now, date_header=0;
int delta;
int compression;
switch(fetch_from_buf_http(conn->inbuf,
&headers, MAX_HEADERS_SIZE,
@ -355,7 +376,8 @@ connection_dir_client_reached_eof(connection_t *conn)
/* case 1, fall through */
}
if(parse_http_response(headers, &status_code, NULL, &date_header) < 0) {
if(parse_http_response(headers, &status_code, NULL, &date_header,
&compression) < 0) {
log_fn(LOG_WARN,"Unparseable headers. Closing.");
free(body); free(headers);
return -1;
@ -372,6 +394,19 @@ connection_dir_client_reached_eof(connection_t *conn)
}
}
if (compression != 0) {
char *new_body;
size_t new_len;
if (tor_gzip_uncompress(&new_body, &new_len, body, body_len, compression)) {
log_fn(LOG_WARN, "Unable to decompress HTTP body.");
tor_free(body); tor_free(headers);
return -1;
}
tor_free(body);
body = new_body;
body_len = (int)new_len;
}
if(conn->purpose == DIR_PURPOSE_FETCH_DIR) {
/* fetch/process the directory to learn about new routers. */
log_fn(LOG_INFO,"Received directory (size %d):\n%s", body_len, body);
@ -545,8 +580,8 @@ directory_handle_command_get(connection_t *conn, char *headers,
return 0;
}
if(!strcmp(url,"/")) { /* directory fetch */
dlen = dirserv_get_directory(&cp, 0);
if(!strcmp(url,"/") || !strcmp(url,"/dir.z")) { /* directory fetch */
dlen = dirserv_get_directory(&cp, !strcmp(url,"/dir.z"));
if(dlen == 0) {
log_fn(LOG_WARN,"My directory is empty. Closing.");
@ -556,9 +591,10 @@ directory_handle_command_get(connection_t *conn, char *headers,
log_fn(LOG_DEBUG,"Dumping directory to client.");
format_rfc1123_time(date, time(NULL));
snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\n\r\n",
snprintf(tmp, sizeof(tmp), "HTTP/1.0 200 OK\r\nDate: %s\r\nContent-Length: %d\r\nContent-Type: text/plain\r\nContent-Encoding: %s\r\n\r\n",
date,
(int)dlen);
(int)dlen,
strcmp(url,"/dir.z")?"identity":"deflate");
connection_write_to_buf(tmp, strlen(tmp), conn);
connection_write_to_buf(cp, strlen(cp), conn);
return 0;

View File

@ -630,10 +630,16 @@ void dirserv_set_cached_directory(const char *directory, time_t when)
{
time_t now;
size_t z_len;
char filename[512];
tor_assert(!options.AuthoritativeDir);
now = time(NULL);
if (when>cached_directory_published &&
when<now+ROUTER_ALLOW_SKEW) {
if (when<=cached_directory_published) {
log_fn(LOG_INFO, "Ignoring old directory; not caching.");
} else if (when>=now+ROUTER_ALLOW_SKEW) {
log_fn(LOG_INFO, "Ignoring future directory; not caching.");
} if (when>cached_directory_published &&
when<now+ROUTER_ALLOW_SKEW) {
log_fn(LOG_DEBUG, "Caching directory.");
tor_free(cached_directory);
cached_directory = tor_strdup(directory);
cached_directory_len = strlen(cached_directory);
@ -644,6 +650,12 @@ void dirserv_set_cached_directory(const char *directory, time_t when)
log_fn(LOG_WARN,"Error compressing cached directory");
}
cached_directory_published = when;
if(get_data_directory(&options)) {
sprintf(filename,"%s/cached-directory", get_data_directory(&options));
if(write_str_to_file(filename,cached_directory) < 0) {
log_fn(LOG_WARN, "Couldn't write cached directory to disk. Ignoring.");
}
}
}
}

View File

@ -21,12 +21,17 @@ extern char *conn_state_to_string[][_CONN_TYPE_MAX+1];
or_options_t options; /**< Command-line and config-file options. */
int global_read_bucket; /**< Max number of bytes I can read this second. */
int global_write_bucket; /**< Max number of bytes I can write this second. */
/** What was the read bucket before the last call to prepare_for_pool?
* (used to determine how many bytes we've read). */
static int stats_prev_global_read_bucket;
/** How many bytes have we read since we started the process? */
/** What was the write bucket before the last call to prepare_for_pool?
* (used to determine how many bytes we've written). */
static int stats_prev_global_write_bucket;
/** How many bytes have we read/written since we started the process? */
static uint64_t stats_n_bytes_read = 0;
static uint64_t stats_n_bytes_written = 0;
/** How many seconds have we been running? */
long stats_n_seconds_uptime = 0;
@ -632,8 +637,10 @@ static int prepare_for_poll(void) {
/* Check how much bandwidth we've consumed, and increment the token
* buckets. */
stats_n_bytes_read += stats_prev_global_read_bucket - global_read_bucket;
stats_n_bytes_written += stats_prev_global_write_bucket - global_write_bucket;
connection_bucket_refill(&now);
stats_prev_global_read_bucket = global_read_bucket;
stats_prev_global_write_bucket = global_write_bucket;
if(now.tv_sec > current_second) { /* the second has rolled over. check more stuff. */
@ -698,6 +705,7 @@ static int init_from_config(int argc, char **argv) {
/* Set up our buckets */
connection_bucket_init();
stats_prev_global_read_bucket = global_read_bucket;
stats_prev_global_write_bucket = global_write_bucket;
/* Finish backgrounding the process */
if(options.RunAsDaemon) {
@ -782,15 +790,8 @@ static int do_main_loop(void) {
}
/* load the routers file, or assign the defaults. */
if(options.RouterFile) {
routerlist_clear_trusted_directories();
if (router_load_routerlist_from_file(options.RouterFile, 1) < 0) {
log_fn(LOG_ERR,"Error loading router list.");
return -1;
}
} else {
if(config_assign_default_dirservers() < 0)
return -1;
if(router_reload_router_list()) {
return -1;
}
if(authdir_mode()) {

View File

@ -867,8 +867,6 @@ typedef struct {
int AuthoritativeDir; /**< Boolean: is this an authoritative directory? */
int ClientOnly; /**< Boolean: should we never evolve into a server role? */
int MaxConn; /**< Maximum number of simultaneous connections. */
int TrafficShaping; /**< Unused. */
int LinkPadding; /**< Unused. */
int IgnoreVersion; /**< If true, run no matter what versions of Tor the
* directory recommends. */
int RunAsDaemon; /**< If true, run in the background. (Unix only) */
@ -1392,6 +1390,7 @@ int is_legal_nickname_or_hexdigest(const char *s);
/********************************* routerlist.c ***************************/
int router_reload_router_list(void);
routerinfo_t *router_pick_directory_server(int requireauth, int requireothers);
int all_directory_servers_down(void);
struct smartlist_t;

View File

@ -40,6 +40,37 @@ static routerlist_t *routerlist = NULL;
extern int has_fetched_directory; /**< from main.c */
/**
* Reload the original list of trusted dirservers, and the most recent
* cached directory (if present).
*/
int router_reload_router_list(void)
{
char filename[512];
routerlist_clear_trusted_directories();
if (options.RouterFile) {
if (router_load_routerlist_from_file(options.RouterFile, 1) < 0) {
log_fn(LOG_ERR,"Error loading router list.");
return -1;
}
} else {
if (config_assign_default_dirservers() < 0)
return -1;
}
if (get_data_directory(&options)) {
char *s;
sprintf(filename,"%s/cached-directory", get_data_directory(&options));
s = read_file_to_str(filename);
if (s) {
log_fn(LOG_INFO, "Loading cached directory from %s", filename);
if (router_load_routerlist_from_string(s, 0) < 0) {
log_fn(LOG_WARN, "Cached directory was unparseable; ignoring.");
}
}
}
return 0;
}
/** Try to find a running dirserver. If there are no running dirservers
* in our routerlist, set all the authoritative ones as running again,
* and pick one. If there are no dirservers at all in our routerlist,
@ -63,12 +94,8 @@ routerinfo_t *router_pick_directory_server(int requireauth, int requireothers) {
options.FascistFirewall ? "reachable" : "known");
has_fetched_directory=0; /* reset it */
routerlist_clear_trusted_directories();
if(options.RouterFile) {
if(router_load_routerlist_from_file(options.RouterFile, 1) < 0)
return NULL;
} else {
if(config_assign_default_dirservers() < 0)
return NULL;
if(router_reload_router_list()) {
return NULL;
}
/* give it one last try */
choice = router_pick_directory_server_impl(requireauth, requireothers, 0);