mirror of
https://gitlab.torproject.org/tpo/core/tor.git
synced 2024-11-10 21:23:58 +01:00
Next batch of memory miserdom: mmap cached-routers file. This is sure to break somewhere.
svn:r6675
This commit is contained in:
parent
fb6d4855ba
commit
b7910202a3
@ -1330,14 +1330,14 @@ handle_getinfo_helper(const char *question, char **answer)
|
||||
if (ri) {
|
||||
const char *body = signed_descriptor_get_body(&ri->cache_info);
|
||||
if (body)
|
||||
*answer = tor_strdup(body);
|
||||
*answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
|
||||
}
|
||||
} else if (!strcmpstart(question, "desc/name/")) {
|
||||
routerinfo_t *ri = router_get_by_nickname(question+strlen("desc/name/"),1);
|
||||
if (ri) {
|
||||
const char *body = signed_descriptor_get_body(&ri->cache_info);
|
||||
if (body)
|
||||
*answer = tor_strdup(body);
|
||||
*answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
|
||||
}
|
||||
} else if (!strcmp(question, "desc/all-recent")) {
|
||||
routerlist_t *routerlist = router_get_routerlist();
|
||||
@ -1347,7 +1347,8 @@ handle_getinfo_helper(const char *question, char **answer)
|
||||
{
|
||||
const char *body = signed_descriptor_get_body(&ri->cache_info);
|
||||
if (body)
|
||||
smartlist_add(sl, tor_strdup(body));
|
||||
smartlist_add(sl,
|
||||
tor_strndup(body, ri->cache_info.signed_descriptor_len));
|
||||
});
|
||||
}
|
||||
*answer = smartlist_join_strings(sl, "", 0, NULL);
|
||||
|
@ -1076,7 +1076,7 @@ connection_dir_client_reached_eof(connection_t *conn)
|
||||
if (which || (conn->requested_resource &&
|
||||
!strcmpstart(conn->requested_resource, "all"))) {
|
||||
/* as we learn from them, we remove them from 'which' */
|
||||
router_load_routers_from_string(body, 0, which);
|
||||
router_load_routers_from_string(body, SAVED_NOWHERE, which);
|
||||
directory_info_has_arrived(time(NULL), 0);
|
||||
}
|
||||
if (which) { /* mark remaining ones as failed */
|
||||
|
@ -481,7 +481,7 @@ dirserv_add_descriptor(const char *desc, const char **msg)
|
||||
*msg = NULL;
|
||||
|
||||
/* Check: is the descriptor syntactically valid? */
|
||||
ri = router_parse_entry_from_string(desc, NULL);
|
||||
ri = router_parse_entry_from_string(desc, NULL, 1);
|
||||
if (!ri) {
|
||||
log_warn(LD_DIRSERV, "Couldn't parse uploaded server descriptor");
|
||||
*msg = "Rejected: Couldn't parse server descriptor.";
|
||||
@ -1808,6 +1808,7 @@ connection_dirserv_add_servers_to_outbuf(connection_t *conn)
|
||||
|
||||
while (smartlist_len(conn->fingerprint_stack) &&
|
||||
buf_datalen(conn->outbuf) < DIRSERV_BUFFER_MIN) {
|
||||
char *body;
|
||||
char *fp = smartlist_pop_last(conn->fingerprint_stack);
|
||||
signed_descriptor_t *sd = NULL;
|
||||
if (by_fp) {
|
||||
@ -1824,13 +1825,14 @@ connection_dirserv_add_servers_to_outbuf(connection_t *conn)
|
||||
tor_free(fp);
|
||||
if (!sd)
|
||||
continue;
|
||||
body = signed_descriptor_get_body(sd);
|
||||
if (conn->zlib_state) {
|
||||
connection_write_to_buf_zlib(
|
||||
conn, conn->zlib_state,
|
||||
sd->signed_descriptor_body, sd->signed_descriptor_len,
|
||||
body, sd->signed_descriptor_len,
|
||||
0);
|
||||
} else {
|
||||
connection_write_to_buf(sd->signed_descriptor_body,
|
||||
connection_write_to_buf(body,
|
||||
sd->signed_descriptor_len,
|
||||
conn);
|
||||
}
|
||||
|
17
src/or/or.h
17
src/or/or.h
@ -762,6 +762,10 @@ typedef struct cached_dir_t {
|
||||
int refcnt; /**< Reference count for this cached_dir_t. */
|
||||
} cached_dir_t;
|
||||
|
||||
typedef enum {
|
||||
SAVED_NOWHERE=0, SAVED_IN_CACHE, SAVED_IN_JOURNAL
|
||||
} saved_location_t;
|
||||
|
||||
/** Information need to cache an onion router's descriptor. */
|
||||
typedef struct signed_descriptor_t {
|
||||
char *signed_descriptor_body;
|
||||
@ -769,7 +773,7 @@ typedef struct signed_descriptor_t {
|
||||
char signed_descriptor_digest[DIGEST_LEN];
|
||||
char identity_digest[DIGEST_LEN];
|
||||
time_t published_on;
|
||||
enum { SAVED_NOWHERE=0, SAVED_IN_CACHE, SAVED_IN_JOURNAL } saved_location;
|
||||
saved_location_t saved_location;
|
||||
off_t saved_offset;
|
||||
} signed_descriptor_t;
|
||||
|
||||
@ -933,6 +937,9 @@ typedef struct {
|
||||
/** List of signed_descriptor_t for older router descriptors we're
|
||||
* caching. */
|
||||
smartlist_t *old_routers;
|
||||
/** DOCDOC */
|
||||
const char *mmap_descriptors;
|
||||
size_t mmap_descriptors_len;
|
||||
} routerlist_t;
|
||||
|
||||
/** Information on router used when extending a circuit. (We don't need a
|
||||
@ -2367,7 +2374,8 @@ int router_add_to_routerlist(routerinfo_t *router, const char **msg,
|
||||
int from_cache, int from_fetch);
|
||||
int router_load_single_router(const char *s, uint8_t purpose,
|
||||
const char **msg);
|
||||
void router_load_routers_from_string(const char *s, int from_cache,
|
||||
void router_load_routers_from_string(const char *s,
|
||||
saved_location_t saved_location,
|
||||
smartlist_t *requested_fingerprints);
|
||||
typedef enum {
|
||||
NS_FROM_CACHE, NS_FROM_DIR, NS_GENERATED
|
||||
@ -2440,7 +2448,7 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
|
||||
crypto_pk_env_t *private_key);
|
||||
int router_parse_list_from_string(const char **s,
|
||||
smartlist_t *dest,
|
||||
int from_cache);
|
||||
saved_location_t saved_location);
|
||||
int router_parse_routerlist_from_directory(const char *s,
|
||||
routerlist_t **dest,
|
||||
crypto_pk_env_t *pkey,
|
||||
@ -2448,7 +2456,8 @@ int router_parse_routerlist_from_directory(const char *s,
|
||||
int write_to_cache);
|
||||
int router_parse_runningrouters(const char *str);
|
||||
int router_parse_directory(const char *str);
|
||||
routerinfo_t *router_parse_entry_from_string(const char *s, const char *end);
|
||||
routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
|
||||
int cache_copy);
|
||||
addr_policy_t *router_parse_addr_policy_from_string(const char *s,
|
||||
int assume_action);
|
||||
version_status_t tor_version_is_obsolete(const char *myversion,
|
||||
|
@ -777,7 +777,10 @@ router_get_my_descriptor(void)
|
||||
const char *body;
|
||||
if (!router_get_my_routerinfo())
|
||||
return NULL;
|
||||
/* Make sure this is nul-terminated. */
|
||||
tor_assert(desc_routerinfo->cache_info.saved_location == SAVED_NOWHERE);
|
||||
body = signed_descriptor_get_body(&desc_routerinfo->cache_info);
|
||||
tor_assert(!body[desc_routerinfo->cache_info.signed_descriptor_len]);
|
||||
log_debug(LD_GENERAL,"my desc is '%s'", body);
|
||||
return body;
|
||||
}
|
||||
@ -1187,7 +1190,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
|
||||
|
||||
#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
|
||||
cp = s_tmp = s_dup = tor_strdup(s);
|
||||
ri_tmp = router_parse_entry_from_string(cp, NULL);
|
||||
ri_tmp = router_parse_entry_from_string(cp, NULL, 1);
|
||||
if (!ri_tmp) {
|
||||
log_err(LD_BUG,
|
||||
"We just generated a router descriptor we can't parse: <<%s>>",
|
||||
|
@ -194,6 +194,21 @@ router_append_to_journal(signed_descriptor_t *desc)
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int
|
||||
_compare_old_routers_by_age(const void **_a, const void **_b)
|
||||
{
|
||||
const signed_descriptor_t *r1 = *_a, *r2 = *_b;
|
||||
return r1->published_on - r2->published_on;
|
||||
}
|
||||
|
||||
static int
|
||||
_compare_routers_by_age(const void **_a, const void **_b)
|
||||
{
|
||||
const routerinfo_t *r1 = *_a, *r2 = *_b;
|
||||
return r1->cache_info.published_on - r2->cache_info.published_on;
|
||||
}
|
||||
|
||||
|
||||
/** If the journal is too long, or if <b>force</b> is true, then atomically
|
||||
* replace the router store with the routers currently in our routerlist, and
|
||||
* clear the journal. Return 0 on success, -1 on failure.
|
||||
@ -223,8 +238,16 @@ router_rebuild_store(int force)
|
||||
chunk_list = smartlist_create();
|
||||
|
||||
for (i = 0; i < 2; ++i) {
|
||||
smartlist_t *lst = (i == 0) ? routerlist->old_routers :
|
||||
routerlist->routers;
|
||||
smartlist_t *lst = smartlist_create();
|
||||
/* We sort the routers by age to enhance locality on disk. */
|
||||
if (i==0) {
|
||||
smartlist_add_all(lst, routerlist->old_routers);
|
||||
smartlist_sort(lst, _compare_old_routers_by_age);
|
||||
} else {
|
||||
smartlist_add_all(lst, routerlist->routers);
|
||||
smartlist_sort(lst, _compare_routers_by_age);
|
||||
}
|
||||
/* Now, add the appropriate members to chunk_list */
|
||||
SMARTLIST_FOREACH(lst, void *, ptr,
|
||||
{
|
||||
signed_descriptor_t *sd = (i==0) ?
|
||||
@ -233,6 +256,7 @@ router_rebuild_store(int force)
|
||||
const char *body = signed_descriptor_get_body(sd);
|
||||
if (!body) {
|
||||
log_warn(LD_BUG, "Bug! No descriptor available for router.");
|
||||
smartlist_free(lst);
|
||||
goto done;
|
||||
}
|
||||
c = tor_malloc(sizeof(sized_chunk_t));
|
||||
@ -240,11 +264,22 @@ router_rebuild_store(int force)
|
||||
c->len = sd->signed_descriptor_len;
|
||||
smartlist_add(chunk_list, c);
|
||||
});
|
||||
smartlist_free(lst);
|
||||
}
|
||||
if (write_chunks_to_file(fname, chunk_list, 0)<0) {
|
||||
log_warn(LD_FS, "Error writing router store to disk.");
|
||||
goto done;
|
||||
}
|
||||
/* Our mmap is now invalid. */
|
||||
if (routerlist->mmap_descriptors) {
|
||||
tor_munmap_file(routerlist->mmap_descriptors,
|
||||
routerlist->mmap_descriptors_len);
|
||||
routerlist->mmap_descriptors =
|
||||
tor_mmap_file(fname, &routerlist->mmap_descriptors_len);
|
||||
if (! routerlist->mmap_descriptors)
|
||||
log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname);
|
||||
}
|
||||
|
||||
for (i = 0; i < 2; ++i) {
|
||||
smartlist_t *lst = (i == 0) ? routerlist->old_routers :
|
||||
routerlist->routers;
|
||||
@ -256,6 +291,8 @@ router_rebuild_store(int force)
|
||||
|
||||
sd->saved_location = SAVED_IN_CACHE;
|
||||
sd->saved_offset = offset;
|
||||
if (routerlist->mmap_descriptors)
|
||||
sd->signed_descriptor_body = NULL;
|
||||
offset += sd->signed_descriptor_len;
|
||||
});
|
||||
}
|
||||
@ -286,31 +323,32 @@ router_reload_router_list(void)
|
||||
{
|
||||
or_options_t *options = get_options();
|
||||
size_t fname_len = strlen(options->DataDirectory)+32;
|
||||
char *fname = tor_malloc(fname_len);
|
||||
char *fname = tor_malloc(fname_len), *contents;
|
||||
struct stat st;
|
||||
int j;
|
||||
|
||||
if (!routerlist)
|
||||
router_get_routerlist(); /* mallocs and inits it in place */
|
||||
|
||||
router_journal_len = router_store_len = 0;
|
||||
|
||||
for (j = 0; j < 2; ++j) {
|
||||
char *contents;
|
||||
tor_snprintf(fname, fname_len,
|
||||
(j==0)?"%s/cached-routers":"%s/cached-routers.new",
|
||||
options->DataDirectory);
|
||||
contents = read_file_to_str(fname, 0);
|
||||
if (contents) {
|
||||
stat(fname, &st);
|
||||
if (j==0)
|
||||
router_store_len = st.st_size;
|
||||
else
|
||||
router_journal_len = st.st_size;
|
||||
router_load_routers_from_string(contents, 1, NULL);
|
||||
tor_free(contents);
|
||||
}
|
||||
tor_snprintf(fname, fname_len, "%s/cached-routers", options->DataDirectory);
|
||||
routerlist->mmap_descriptors =
|
||||
tor_mmap_file(fname, &routerlist->mmap_descriptors_len);
|
||||
if (routerlist->mmap_descriptors) {
|
||||
router_store_len = routerlist->mmap_descriptors_len;
|
||||
router_load_routers_from_string(routerlist->mmap_descriptors,
|
||||
SAVED_IN_CACHE, NULL);
|
||||
}
|
||||
|
||||
tor_snprintf(fname, fname_len, "%s/cached-routers.new",
|
||||
options->DataDirectory);
|
||||
contents = read_file_to_str(fname, 0);
|
||||
if (contents) {
|
||||
stat(fname, &st);
|
||||
router_load_routers_from_string(contents,
|
||||
SAVED_IN_JOURNAL, NULL);
|
||||
}
|
||||
|
||||
tor_free(fname);
|
||||
|
||||
if (router_journal_len) {
|
||||
@ -1103,10 +1141,26 @@ router_get_by_descriptor_digest(const char *digest)
|
||||
return digestmap_get(routerlist->desc_digest_map, digest);
|
||||
}
|
||||
|
||||
/* DOCDOC Not always nul-terminated. */
|
||||
const char *
|
||||
signed_descriptor_get_body(signed_descriptor_t *desc)
|
||||
{
|
||||
return desc->signed_descriptor_body;
|
||||
const char *r;
|
||||
size_t len = desc->signed_descriptor_len;
|
||||
tor_assert(len > 32);
|
||||
if (desc->saved_location == SAVED_IN_CACHE && routerlist &&
|
||||
routerlist->mmap_descriptors) {
|
||||
tor_assert(desc->saved_offset + len <= routerlist->mmap_descriptors_len);
|
||||
r = routerlist->mmap_descriptors + desc->saved_offset;
|
||||
} else {
|
||||
r = desc->signed_descriptor_body;
|
||||
}
|
||||
tor_assert(r);
|
||||
tor_assert(!memcmp("router ", r, 7));
|
||||
tor_assert(!memcmp("\n-----END SIGNATURE-----\n",
|
||||
r + len - 25, 25));
|
||||
|
||||
return r;
|
||||
}
|
||||
|
||||
/** Return the current list of all known routers. */
|
||||
@ -1887,7 +1941,7 @@ router_load_single_router(const char *s, uint8_t purpose, const char **msg)
|
||||
tor_assert(msg);
|
||||
*msg = NULL;
|
||||
|
||||
if (!(ri = router_parse_entry_from_string(s, NULL))) {
|
||||
if (!(ri = router_parse_entry_from_string(s, NULL, 1))) {
|
||||
log_warn(LD_DIR, "Error parsing router descriptor; dropping.");
|
||||
*msg = "Couldn't parse router descriptor.";
|
||||
return -1;
|
||||
@ -1927,16 +1981,18 @@ router_load_single_router(const char *s, uint8_t purpose, const char **msg)
|
||||
* uppercased identity fingerprints. Do not update any router whose
|
||||
* fingerprint is not on the list; after updating a router, remove its
|
||||
* fingerprint from the list.
|
||||
* DOCDOC saved_location
|
||||
*/
|
||||
void
|
||||
router_load_routers_from_string(const char *s, int from_cache,
|
||||
router_load_routers_from_string(const char *s, saved_location_t saved_location,
|
||||
smartlist_t *requested_fingerprints)
|
||||
{
|
||||
smartlist_t *routers = smartlist_create(), *changed = smartlist_create();
|
||||
char fp[HEX_DIGEST_LEN+1];
|
||||
const char *msg;
|
||||
int from_cache = (saved_location != SAVED_NOWHERE);
|
||||
|
||||
router_parse_list_from_string(&s, routers, from_cache);
|
||||
router_parse_list_from_string(&s, routers, saved_location);
|
||||
|
||||
routers_update_status_from_networkstatus(routers, !from_cache);
|
||||
|
||||
|
@ -635,10 +635,11 @@ check_directory_signature(const char *digest,
|
||||
* are marked running and valid. Advances *s to a point immediately
|
||||
* following the last router entry. Ignore any trailing router entries that
|
||||
* are not complete. Returns 0 on success and -1 on failure.
|
||||
* DOCDOC saved_location
|
||||
*/
|
||||
int
|
||||
router_parse_list_from_string(const char **s, smartlist_t *dest,
|
||||
int from_cache)
|
||||
saved_location_t saved_location)
|
||||
{
|
||||
routerinfo_t *router;
|
||||
const char *end, *cp, *start;
|
||||
@ -678,15 +679,16 @@ router_parse_list_from_string(const char **s, smartlist_t *dest,
|
||||
continue;
|
||||
}
|
||||
|
||||
router = router_parse_entry_from_string(*s, end);
|
||||
router = router_parse_entry_from_string(*s, end,
|
||||
saved_location != SAVED_IN_CACHE);
|
||||
|
||||
*s = end;
|
||||
if (!router) {
|
||||
log_warn(LD_DIR, "Error reading router; skipping");
|
||||
continue;
|
||||
}
|
||||
if (from_cache) {
|
||||
router->cache_info.saved_location = SAVED_IN_CACHE;
|
||||
if (saved_location != SAVED_NOWHERE) {
|
||||
router->cache_info.saved_location = saved_location;
|
||||
router->cache_info.saved_offset = *s - start;
|
||||
}
|
||||
smartlist_add(dest, router);
|
||||
@ -698,9 +700,11 @@ router_parse_list_from_string(const char **s, smartlist_t *dest,
|
||||
/** Helper function: reads a single router entry from *<b>s</b> ...
|
||||
* *<b>end</b>. Mallocs a new router and returns it if all goes well, else
|
||||
* returns NULL.
|
||||
* DOCDOC cache_copy
|
||||
*/
|
||||
routerinfo_t *
|
||||
router_parse_entry_from_string(const char *s, const char *end)
|
||||
router_parse_entry_from_string(const char *s, const char *end,
|
||||
int cache_copy)
|
||||
{
|
||||
routerinfo_t *router = NULL;
|
||||
char signed_digest[128];
|
||||
@ -747,7 +751,8 @@ router_parse_entry_from_string(const char *s, const char *end)
|
||||
}
|
||||
|
||||
router = tor_malloc_zero(sizeof(routerinfo_t));
|
||||
router->cache_info.signed_descriptor_body = tor_strndup(s, end-s);
|
||||
if (cache_copy)
|
||||
router->cache_info.signed_descriptor_body = tor_strndup(s, end-s);
|
||||
router->cache_info.signed_descriptor_len = end-s;
|
||||
memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);
|
||||
|
||||
|
@ -1297,7 +1297,7 @@ test_dir_format(void)
|
||||
|
||||
test_assert(router_dump_router_to_string(buf, 2048, &r1, pk2)>0);
|
||||
cp = buf;
|
||||
rp1 = router_parse_entry_from_string((const char*)cp,NULL);
|
||||
rp1 = router_parse_entry_from_string((const char*)cp,NULL,1);
|
||||
test_assert(rp1);
|
||||
test_streq(rp1->address, r1.address);
|
||||
test_eq(rp1->or_port, r1.or_port);
|
||||
@ -1320,7 +1320,7 @@ test_dir_format(void)
|
||||
test_streq(buf, buf2);
|
||||
|
||||
cp = buf;
|
||||
rp2 = router_parse_entry_from_string(&cp);
|
||||
rp2 = router_parse_entry_from_string(&cp,1);
|
||||
test_assert(rp2);
|
||||
test_streq(rp2->address, r2.address);
|
||||
test_eq(rp2->or_port, r2.or_port);
|
||||
|
Loading…
Reference in New Issue
Block a user