nihilist/tor
1
0
Fork 0
mirror of https://gitlab.torproject.org/tpo/core/tor.git synced 2024-11-24 12:23:32 +01:00
Code Issues Packages Projects Releases Wiki Activity

Next batch of memory miserdom: mmap cached-routers file. This is sure to break somewhere.

Browse Source 
svn:r6675
This commit is contained in:
Nick Mathewson 2006-06-22 07:01:54 +00:00
parent fb6d4855ba
commit b7910202a3
8 changed files with 119 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
src/or/control.c
View File

@ -1330,14 +1330,14 @@ handle_getinfo_helper(const char *question, char **answer)
if (ri) {
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
*answer = tor_strdup(body);
*answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
}
} else if (!strcmpstart(question, "desc/name/")) {
routerinfo_t *ri = router_get_by_nickname(question+strlen("desc/name/"),1);
if (ri) {
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
*answer = tor_strdup(body);
*answer = tor_strndup(body, ri->cache_info.signed_descriptor_len);
}
} else if (!strcmp(question, "desc/all-recent")) {
routerlist_t *routerlist = router_get_routerlist();
@ -1347,7 +1347,8 @@ handle_getinfo_helper(const char *question, char **answer)
{
const char *body = signed_descriptor_get_body(&ri->cache_info);
if (body)
smartlist_add(sl, tor_strdup(body));
smartlist_add(sl,
tor_strndup(body, ri->cache_info.signed_descriptor_len));
});
}
*answer = smartlist_join_strings(sl, "", 0, NULL);

2
src/or/directory.c
View File

@ -1076,7 +1076,7 @@ connection_dir_client_reached_eof(connection_t *conn)
if (which || (conn->requested_resource &&
!strcmpstart(conn->requested_resource, "all"))) {
/* as we learn from them, we remove them from 'which' */
router_load_routers_from_string(body, 0, which);
router_load_routers_from_string(body, SAVED_NOWHERE, which);
directory_info_has_arrived(time(NULL), 0);
}
if (which) { /* mark remaining ones as failed */

8
src/or/dirserv.c
View File

@ -481,7 +481,7 @@ dirserv_add_descriptor(const char *desc, const char **msg)
*msg = NULL;
/* Check: is the descriptor syntactically valid? */
ri = router_parse_entry_from_string(desc, NULL);
ri = router_parse_entry_from_string(desc, NULL, 1);
if (!ri) {
log_warn(LD_DIRSERV, "Couldn't parse uploaded server descriptor");
*msg = "Rejected: Couldn't parse server descriptor.";
@ -1808,6 +1808,7 @@ connection_dirserv_add_servers_to_outbuf(connection_t *conn)
while (smartlist_len(conn->fingerprint_stack) &&
buf_datalen(conn->outbuf) < DIRSERV_BUFFER_MIN) {
char *body;
char *fp = smartlist_pop_last(conn->fingerprint_stack);
signed_descriptor_t *sd = NULL;
if (by_fp) {
@ -1824,13 +1825,14 @@ connection_dirserv_add_servers_to_outbuf(connection_t *conn)
tor_free(fp);
if (!sd)
continue;
body = signed_descriptor_get_body(sd);
if (conn->zlib_state) {
connection_write_to_buf_zlib(
conn, conn->zlib_state,
sd->signed_descriptor_body, sd->signed_descriptor_len,
body, sd->signed_descriptor_len,
0);
} else {
connection_write_to_buf(sd->signed_descriptor_body,
connection_write_to_buf(body,
sd->signed_descriptor_len,
conn);
}

17
src/or/or.h
View File

@ -762,6 +762,10 @@ typedef struct cached_dir_t {
int refcnt; /**< Reference count for this cached_dir_t. */
} cached_dir_t;
typedef enum {
SAVED_NOWHERE=0, SAVED_IN_CACHE, SAVED_IN_JOURNAL
} saved_location_t;
/** Information need to cache an onion router's descriptor. */
typedef struct signed_descriptor_t {
char *signed_descriptor_body;
@ -769,7 +773,7 @@ typedef struct signed_descriptor_t {
char signed_descriptor_digest[DIGEST_LEN];
char identity_digest[DIGEST_LEN];
time_t published_on;
enum { SAVED_NOWHERE=0, SAVED_IN_CACHE, SAVED_IN_JOURNAL } saved_location;
saved_location_t saved_location;
off_t saved_offset;
} signed_descriptor_t;
@ -933,6 +937,9 @@ typedef struct {
/** List of signed_descriptor_t for older router descriptors we're
* caching. */
smartlist_t *old_routers;
/** DOCDOC */
const char *mmap_descriptors;
size_t mmap_descriptors_len;
} routerlist_t;
/** Information on router used when extending a circuit. (We don't need a
@ -2367,7 +2374,8 @@ int router_add_to_routerlist(routerinfo_t *router, const char **msg,
int from_cache, int from_fetch);
int router_load_single_router(const char *s, uint8_t purpose,
const char **msg);
void router_load_routers_from_string(const char *s, int from_cache,
void router_load_routers_from_string(const char *s,
saved_location_t saved_location,
smartlist_t *requested_fingerprints);
typedef enum {
NS_FROM_CACHE, NS_FROM_DIR, NS_GENERATED
@ -2440,7 +2448,7 @@ int router_append_dirobj_signature(char *buf, size_t buf_len,
crypto_pk_env_t *private_key);
int router_parse_list_from_string(const char **s,
smartlist_t *dest,
int from_cache);
saved_location_t saved_location);
int router_parse_routerlist_from_directory(const char *s,
routerlist_t **dest,
crypto_pk_env_t *pkey,
@ -2448,7 +2456,8 @@ int router_parse_routerlist_from_directory(const char *s,
int write_to_cache);
int router_parse_runningrouters(const char *str);
int router_parse_directory(const char *str);
routerinfo_t *router_parse_entry_from_string(const char *s, const char *end);
routerinfo_t *router_parse_entry_from_string(const char *s, const char *end,
int cache_copy);
addr_policy_t *router_parse_addr_policy_from_string(const char *s,
int assume_action);
version_status_t tor_version_is_obsolete(const char *myversion,

5
src/or/router.c
View File

@ -777,7 +777,10 @@ router_get_my_descriptor(void)
const char *body;
if (!router_get_my_routerinfo())
return NULL;
/* Make sure this is nul-terminated. */
tor_assert(desc_routerinfo->cache_info.saved_location == SAVED_NOWHERE);
body = signed_descriptor_get_body(&desc_routerinfo->cache_info);
tor_assert(!body[desc_routerinfo->cache_info.signed_descriptor_len]);
log_debug(LD_GENERAL,"my desc is '%s'", body);
return body;
}
@ -1187,7 +1190,7 @@ router_dump_router_to_string(char *s, size_t maxlen, routerinfo_t *router,
#ifdef DEBUG_ROUTER_DUMP_ROUTER_TO_STRING
cp = s_tmp = s_dup = tor_strdup(s);
ri_tmp = router_parse_entry_from_string(cp, NULL);
ri_tmp = router_parse_entry_from_string(cp, NULL, 1);
if (!ri_tmp) {
log_err(LD_BUG,
"We just generated a router descriptor we can't parse: <<%s>>",

102
src/or/routerlist.c
View File

@ -194,6 +194,21 @@ router_append_to_journal(signed_descriptor_t *desc)
return 0;
}
static int
_compare_old_routers_by_age(const void **_a, const void **_b)
{
const signed_descriptor_t *r1 = *_a, *r2 = *_b;
return r1->published_on - r2->published_on;
}
static int
_compare_routers_by_age(const void **_a, const void **_b)
{
const routerinfo_t *r1 = *_a, *r2 = *_b;
return r1->cache_info.published_on - r2->cache_info.published_on;
}
/** If the journal is too long, or if <b>force</b> is true, then atomically
* replace the router store with the routers currently in our routerlist, and
* clear the journal. Return 0 on success, -1 on failure.
@ -223,8 +238,16 @@ router_rebuild_store(int force)
chunk_list = smartlist_create();
for (i = 0; i < 2; ++i) {
smartlist_t *lst = (i == 0) ? routerlist->old_routers :
routerlist->routers;
smartlist_t *lst = smartlist_create();
/* We sort the routers by age to enhance locality on disk. */
if (i==0) {
smartlist_add_all(lst, routerlist->old_routers);
smartlist_sort(lst, _compare_old_routers_by_age);
} else {
smartlist_add_all(lst, routerlist->routers);
smartlist_sort(lst, _compare_routers_by_age);
}
/* Now, add the appropriate members to chunk_list */
SMARTLIST_FOREACH(lst, void *, ptr,
{
signed_descriptor_t *sd = (i==0) ?
@ -233,6 +256,7 @@ router_rebuild_store(int force)
const char *body = signed_descriptor_get_body(sd);
if (!body) {
log_warn(LD_BUG, "Bug! No descriptor available for router.");
smartlist_free(lst);
goto done;
}
c = tor_malloc(sizeof(sized_chunk_t));
@ -240,11 +264,22 @@ router_rebuild_store(int force)
c->len = sd->signed_descriptor_len;
smartlist_add(chunk_list, c);
});
smartlist_free(lst);
}
if (write_chunks_to_file(fname, chunk_list, 0)<0) {
log_warn(LD_FS, "Error writing router store to disk.");
goto done;
}
/* Our mmap is now invalid. */
if (routerlist->mmap_descriptors) {
tor_munmap_file(routerlist->mmap_descriptors,
routerlist->mmap_descriptors_len);
routerlist->mmap_descriptors =
tor_mmap_file(fname, &routerlist->mmap_descriptors_len);
if (! routerlist->mmap_descriptors)
log_warn(LD_FS, "Unable to mmap new descriptor file at '%s'.",fname);
}
for (i = 0; i < 2; ++i) {
smartlist_t *lst = (i == 0) ? routerlist->old_routers :
routerlist->routers;
@ -256,6 +291,8 @@ router_rebuild_store(int force)
sd->saved_location = SAVED_IN_CACHE;
sd->saved_offset = offset;
if (routerlist->mmap_descriptors)
sd->signed_descriptor_body = NULL;
offset += sd->signed_descriptor_len;
});
}
@ -286,31 +323,32 @@ router_reload_router_list(void)
{
or_options_t *options = get_options();
size_t fname_len = strlen(options->DataDirectory)+32;
char *fname = tor_malloc(fname_len);
char *fname = tor_malloc(fname_len), *contents;
struct stat st;
int j;
if (!routerlist)
router_get_routerlist(); /* mallocs and inits it in place */
router_journal_len = router_store_len = 0;
for (j = 0; j < 2; ++j) {
char *contents;
tor_snprintf(fname, fname_len,
(j==0)?"%s/cached-routers":"%s/cached-routers.new",
options->DataDirectory);
contents = read_file_to_str(fname, 0);
if (contents) {
stat(fname, &st);
if (j==0)
router_store_len = st.st_size;
else
router_journal_len = st.st_size;
router_load_routers_from_string(contents, 1, NULL);
tor_free(contents);
}
tor_snprintf(fname, fname_len, "%s/cached-routers", options->DataDirectory);
routerlist->mmap_descriptors =
tor_mmap_file(fname, &routerlist->mmap_descriptors_len);
if (routerlist->mmap_descriptors) {
router_store_len = routerlist->mmap_descriptors_len;
router_load_routers_from_string(routerlist->mmap_descriptors,
SAVED_IN_CACHE, NULL);
}
tor_snprintf(fname, fname_len, "%s/cached-routers.new",
options->DataDirectory);
contents = read_file_to_str(fname, 0);
if (contents) {
stat(fname, &st);
router_load_routers_from_string(contents,
SAVED_IN_JOURNAL, NULL);
}
tor_free(fname);
if (router_journal_len) {
@ -1103,10 +1141,26 @@ router_get_by_descriptor_digest(const char *digest)
return digestmap_get(routerlist->desc_digest_map, digest);
}
/* DOCDOC Not always nul-terminated. */
const char *
signed_descriptor_get_body(signed_descriptor_t *desc)
{
return desc->signed_descriptor_body;
const char *r;
size_t len = desc->signed_descriptor_len;
tor_assert(len > 32);
if (desc->saved_location == SAVED_IN_CACHE && routerlist &&
routerlist->mmap_descriptors) {
tor_assert(desc->saved_offset + len <= routerlist->mmap_descriptors_len);
r = routerlist->mmap_descriptors + desc->saved_offset;
} else {
r = desc->signed_descriptor_body;
}
tor_assert(r);
tor_assert(!memcmp("router ", r, 7));
tor_assert(!memcmp("\n-----END SIGNATURE-----\n",
r + len - 25, 25));
return r;
}
/** Return the current list of all known routers. */
@ -1887,7 +1941,7 @@ router_load_single_router(const char *s, uint8_t purpose, const char **msg)
tor_assert(msg);
*msg = NULL;
if (!(ri = router_parse_entry_from_string(s, NULL))) {
if (!(ri = router_parse_entry_from_string(s, NULL, 1))) {
log_warn(LD_DIR, "Error parsing router descriptor; dropping.");
*msg = "Couldn't parse router descriptor.";
return -1;
@ -1927,16 +1981,18 @@ router_load_single_router(const char *s, uint8_t purpose, const char **msg)
* uppercased identity fingerprints. Do not update any router whose
* fingerprint is not on the list; after updating a router, remove its
* fingerprint from the list.
* DOCDOC saved_location
*/
void
router_load_routers_from_string(const char *s, int from_cache,
router_load_routers_from_string(const char *s, saved_location_t saved_location,
smartlist_t *requested_fingerprints)
{
smartlist_t *routers = smartlist_create(), *changed = smartlist_create();
char fp[HEX_DIGEST_LEN+1];
const char *msg;
int from_cache = (saved_location != SAVED_NOWHERE);
router_parse_list_from_string(&s, routers, from_cache);
router_parse_list_from_string(&s, routers, saved_location);
routers_update_status_from_networkstatus(routers, !from_cache);

17
src/or/routerparse.c
View File

@ -635,10 +635,11 @@ check_directory_signature(const char *digest,
* are marked running and valid. Advances *s to a point immediately
* following the last router entry. Ignore any trailing router entries that
* are not complete. Returns 0 on success and -1 on failure.
* DOCDOC saved_location
*/
int
router_parse_list_from_string(const char **s, smartlist_t *dest,
int from_cache)
saved_location_t saved_location)
{
routerinfo_t *router;
const char *end, *cp, *start;
@ -678,15 +679,16 @@ router_parse_list_from_string(const char **s, smartlist_t *dest,
continue;
}
router = router_parse_entry_from_string(*s, end);
router = router_parse_entry_from_string(*s, end,
saved_location != SAVED_IN_CACHE);
*s = end;
if (!router) {
log_warn(LD_DIR, "Error reading router; skipping");
continue;
}
if (from_cache) {
router->cache_info.saved_location = SAVED_IN_CACHE;
if (saved_location != SAVED_NOWHERE) {
router->cache_info.saved_location = saved_location;
router->cache_info.saved_offset = *s - start;
}
smartlist_add(dest, router);
@ -698,9 +700,11 @@ router_parse_list_from_string(const char **s, smartlist_t *dest,
/** Helper function: reads a single router entry from *<b>s</b> ...
* *<b>end</b>. Mallocs a new router and returns it if all goes well, else
* returns NULL.
* DOCDOC cache_copy
*/
routerinfo_t *
router_parse_entry_from_string(const char *s, const char *end)
router_parse_entry_from_string(const char *s, const char *end,
int cache_copy)
{
routerinfo_t *router = NULL;
char signed_digest[128];
@ -747,7 +751,8 @@ router_parse_entry_from_string(const char *s, const char *end)
}
router = tor_malloc_zero(sizeof(routerinfo_t));
router->cache_info.signed_descriptor_body = tor_strndup(s, end-s);
if (cache_copy)
router->cache_info.signed_descriptor_body = tor_strndup(s, end-s);
router->cache_info.signed_descriptor_len = end-s;
memcpy(router->cache_info.signed_descriptor_digest, digest, DIGEST_LEN);

4
src/or/test.c
View File

@ -1297,7 +1297,7 @@ test_dir_format(void)
test_assert(router_dump_router_to_string(buf, 2048, &r1, pk2)>0);
cp = buf;
rp1 = router_parse_entry_from_string((const char*)cp,NULL);
rp1 = router_parse_entry_from_string((const char*)cp,NULL,1);
test_assert(rp1);
test_streq(rp1->address, r1.address);
test_eq(rp1->or_port, r1.or_port);
@ -1320,7 +1320,7 @@ test_dir_format(void)
test_streq(buf, buf2);
cp = buf;
rp2 = router_parse_entry_from_string(&cp);
rp2 = router_parse_entry_from_string(&cp,1);
test_assert(rp2);
test_streq(rp2->address, r2.address);
test_eq(rp2->or_port, r2.or_port);